diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..83053262f6235419331194377c371560da16c170 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..9603dc849ab72dbe2e0b93c9f04a4bc367d9ce11 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ec6edf76939461f2441f7833932f3bdcc2606272 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..5ed55860ed7a588f6855295835f5e69f1f1df502 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..47916f8947bf9211fb831155b90ef5bf0f8f820f Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..90935ea69f1c563b02675097e62e625abb93a796 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d5d4e68e3de3a359ab0ef7ce8cebf1ce3a4df0cc Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..9fc955d60219119f227ac90b877d66fe5a396bf4 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..50f42d90ad0dfab85fa955559ccab37109b9f52d Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..5ccad5a62ec1dc21108c6f8cfd028fa1b3dfd364 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..26f237a2cb0201dd52c137eb7509193c0c8d8021 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..3e5dc3f358bcd35a219c2e655be7459c540ec774 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/plots_step_41000/cld_edm_z_tautau_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/runs/train/events.out.tfevents.1794727308.workergpu169.143507.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/runs/train/events.out.tfevents.1794727308.workergpu169.143507.0 new file mode 100644 index 0000000000000000000000000000000000000000..8af8ef6b7a539eb170f1db2085a403e0c1e46986 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/runs/train/events.out.tfevents.1794727308.workergpu169.143507.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b8ef59c2c107230f7fb7818a790445f0312b2ae4d1a1007d9af3f629a24e33b +size 1510264 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/runs/valid/events.out.tfevents.1794727308.workergpu169.143507.1 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/runs/valid/events.out.tfevents.1794727308.workergpu169.143507.1 new file mode 100644 index 0000000000000000000000000000000000000000..5e4f5d1b899a3465ce746ef9bd5204a11ca01622 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/runs/valid/events.out.tfevents.1794727308.workergpu169.143507.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b9a8f5619c0c2c3b07e0e400dadffe8f54681e971f487d62c7e8dd5b7c92322 +size 18880598 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..55008a1e3175a290aa890f488f61ea278219a48e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/scenario-manifest.json @@ -0,0 +1,1280 @@ +{ + "scenario": { + "name": "pf_fm_finetuning", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all" + }, + "allowed_variant_differences": [ + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets", + "data_config" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_finetuning", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_ecal_2mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all", + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 30, + "checkpoint_freq": 2000, + "val_freq": 1000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "relaxed_load": false, + "load_weights_only": true, + "reset_task_loss_weights": true, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_z_tautau_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--load", + "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "1000", + "--checkpoint_freq", + "2000", + "--patience", + "30", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--load_weights_only", + "true", + "--reset_task_loss_weights", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.feature_bag", + "true", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.trainable", + "all", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0bebc90b5d26c1fe569d4aeda295d854788c6c48 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_z_tautau_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +load_weights_only: true +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 30 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: true +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_z_tautau_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 1000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/train.log b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/train.log new file mode 100644 index 0000000000000000000000000000000000000000..a61962051828a797954f91b0c59abbdfd65a710e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/train.log @@ -0,0 +1,3166 @@ +rank0 [2026-11-15 02:21:44,796] INFO : Will use single-gpu: NVIDIA H100 PCIe +rank0 [2026-11-15 02:21:44,796] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-15 02:21:45,906] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 02:21:45,906] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 02:21:45,907] INFO : Instantiating model +rank0 [2026-11-15 02:21:45,933] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 02:21:45,933] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 02:21:45,935] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 02:21:45,935] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 02:21:45,935] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 02:21:45,935] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 02:21:45,935] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,938] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 02:21:45,938] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,940] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 02:21:45,940] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,943] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 02:21:45,943] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,946] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 02:21:45,946] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,948] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 02:21:45,948] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,951] INFO : Backbone initialization took 0.02s +rank0 [2026-11-15 02:21:45,951] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 02:21:45,951] INFO : Initializing output DNNs +rank0 [2026-11-15 02:21:45,965] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,968] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,975] INFO : Output DNNs initialization took 0.02s +rank0 [2026-11-15 02:21:45,976] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 02:21:45,976] INFO : jagged_attention enabled=True +rank0 [2026-11-15 02:21:45,976] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 02:21:45,976] INFO : task_queries enabled=True +rank0 [2026-11-15 02:21:45,976] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 02:21:45,976] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 02:21:45,976] INFO : classification_query parameters: 256 +rank0 [2026-11-15 02:21:45,976] INFO : regression_query parameters: 256 +rank0 [2026-11-15 02:21:45,976] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 02:21:45,976] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 02:21:45,976] INFO : output_mode=elementwise +rank0 [2026-11-15 02:21:45,976] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 02:21:45,976] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 02:21:45,976] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 02:21:45,976] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 02:21:45,976] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 02:21:45,976] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 02:21:45,976] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 02:21:45,977] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 02:21:45,978] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 02:21:45,978] INFO : MLPF __init__ done +rank0 [2026-11-15 02:21:45,978] INFO : Instantiated model +rank0 [2026-11-15 02:21:45,978] INFO : Moving model to device rank=0 +rank0 [2026-11-15 02:21:45,989] INFO : Moved model to device rank=0 +rank0 [2026-11-15 02:21:45,990] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 02:21:46,117] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 02:21:46,117] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 02:21:46,117] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 02:21:46,215] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 02:21:46,215] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 02:21:46,216] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 02:21:46,216] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 02:21:46,216] INFO : Total parameters: 3910930 +rank0 [2026-11-15 02:21:46,218] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 02:21:46,224] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398 +rank0 [2026-11-15 02:21:46,224] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398 +rank0 [2026-11-15 02:21:46,224] INFO : Getting dataloaders +rank0 [2026-11-15 02:21:46,224] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:21:48,105] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,160] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 02:21:48,328] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 02:21:48,410] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,451] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 02:21:48,528] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,573] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,606] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,656] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 02:21:48,695] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,695] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 02:21:48,696] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 02:21:48,696] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-15 02:21:48,696] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:21:48,703] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,710] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,717] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,724] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,731] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,738] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,746] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,752] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,763] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,770] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,770] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 02:21:48,770] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 02:21:48,799] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:21:48,950] INFO : Got dataloaders +rank0 [2026-11-15 02:21:48,951] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-15 02:21:48,951] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-15 02:21:48,951] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 02:21:48,958] INFO : Creating train_iterator +rank0 [2026-11-15 02:21:48,958] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:21:48,983] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:21:49,139] INFO : Created train_iterator +rank0 [2026-11-15 02:22:01,445] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.7744583451747895, 'Classification': 0.023724770015105603, 'Regression_pt': 0.21244246032088995, 'Regression_eta': 0.0048217891092645, 'Regression_sin_phi': 0.001596466990886256, 'Regression_cos_phi': 0.0022173133328033147, 'Regression_energy': 0.22335495825856924}, weights={'Classification_binary': 1.0, 'Classification': 74.79348754882812, 'Regression_pt': 8.352653503417969, 'Regression_eta': 368.00830078125, 'Regression_sin_phi': 1000.0, 'Regression_cos_phi': 800.2740478515625, 'Regression_energy': 7.94456672668457} +rank0 [2026-11-15 02:22:50,708] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 02:22:50,708] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:23:37,470] INFO : Step 1000/500000 rank0 | Train Loss: 5.6218 | Train Loss Unweighted: 0.9890 | Train Loss Components: Classification: 0.0060 | Classification_binary: 0.5879 | Regression_cos_phi: 0.0004 | Regression_energy: 0.1936 | Regression_eta: 0.0019 | Regression_pt: 0.1989 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.4469 | Classification_binary: 0.5879 | Regression_cos_phi: 0.3096 | Regression_energy: 1.5382 | Regression_eta: 0.6922 | Regression_pt: 1.6612 | Regression_sin_phi: 0.3858 | LR: 2.01e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4571s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0002 | max_forward: 0.0362 | max_loss: 0.0028 | max_backward: 0.0404 | max_optimizer_step: 0.0561 | max_tensorboard_logging: 0.1385 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2577.5 | uss_MB: 1341.7 | child_rss_MB: 22955.9 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:23:37,471] INFO : Running validation on rank0 +rank0 [2026-11-15 02:23:37,472] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:23:37,472] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:23:37,610] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 02:23:37,793] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:23:37,860] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:23:37,912] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:23:37,965] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:23:38,011] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:23:38,065] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:23:38,119] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:23:38,168] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:23:38,232] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:23:38,297] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:23:38,349] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:23:38,403] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:23:38,449] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:23:38,504] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:23:38,558] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:23:38,608] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:23:38,611] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:23:38,614] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:23:38,616] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:23:38,618] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:23:38,620] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:23:38,621] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:23:38,623] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:23:38,623] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:23:38,626] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:23:38,627] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:23:38,630] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:23:38,631] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:23:38,633] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:23:38,633] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:23:38,637] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:23:40,100] INFO : VALIDATION | Step=1000/500000 | Train Loss=5.6218 | Valid Loss=5.4613 | Train Loss Unweighted=0.9890 | Valid Loss Unweighted=1.2668 | Stale=0 | ETA=924.3m +rank0 [2026-11-15 02:23:40,101] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:23:40,175] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:23:40,180] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:23:40,227] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:23:40,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:23:41,221] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:23:41,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:23:41,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:23:41,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:23:42,188] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:23:42,432] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:23:42,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:23:42,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:23:43,149] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:23:43,399] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:23:43,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:23:43,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:23:44,113] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:23:44,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:23:44,562] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:23:44,609] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:23:44,615] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:23:45,151] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:23:45,988] INFO : Plotted number of elements +rank0 [2026-11-15 02:23:47,270] INFO : Plotted jets +rank0 [2026-11-15 02:23:48,560] INFO : Plotted jet ratio +rank0 [2026-11-15 02:24:01,953] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 02:24:01,953] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:25:01,305] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 02:25:01,305] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:25:34,810] INFO : Step 2000/500000 rank0 | Train Loss: 3.1491 | Train Loss Unweighted: 1.3639 | Train Loss Components: Classification: 0.0031 | Classification_binary: 1.2445 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0575 | Regression_eta: 0.0004 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.2287 | Classification_binary: 1.2445 | Regression_cos_phi: 0.4398 | Regression_energy: 0.4572 | Regression_eta: 0.1290 | Regression_pt: 0.4828 | Regression_sin_phi: 0.1671 | LR: 2.02e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4497s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0380 | max_optimizer_step: 0.0570 | max_tensorboard_logging: 0.1291 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2698.8 | uss_MB: 1470.4 | child_rss_MB: 22746.4 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:25:34,812] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 02:25:34,879] INFO : Running validation on rank0 +rank0 [2026-11-15 02:25:34,879] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:25:34,879] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:25:34,996] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 02:25:35,103] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:25:35,202] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:25:35,269] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:25:35,333] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:25:35,396] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:25:35,461] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:25:35,526] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:25:35,633] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:25:35,732] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:25:35,830] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:25:35,896] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:25:35,962] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:25:36,028] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:25:36,095] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:25:36,160] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:25:36,269] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:25:36,276] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:25:36,278] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:25:36,280] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:25:36,281] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:25:36,284] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:25:36,285] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:25:36,285] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:25:36,288] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:25:36,289] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:25:36,292] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:25:36,294] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:25:36,295] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:25:36,295] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:25:36,299] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:25:36,301] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:25:37,753] INFO : VALIDATION | Step=2000/500000 | Train Loss=3.1491 | Valid Loss=5.3821 | Train Loss Unweighted=1.3639 | Valid Loss Unweighted=1.2212 | Stale=0 | ETA=949.5m +rank0 [2026-11-15 02:25:37,753] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:25:37,828] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:25:37,835] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:25:37,878] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:25:38,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:25:38,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:25:38,827] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:25:39,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:25:39,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:25:39,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:25:39,783] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:25:40,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:25:40,253] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:25:40,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:25:40,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:25:40,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:25:41,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:25:41,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:25:41,698] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:25:41,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:25:41,960] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:25:41,965] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:25:42,491] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:25:42,838] INFO : Plotted number of elements +rank0 [2026-11-15 02:25:44,458] INFO : Plotted jets +rank0 [2026-11-15 02:25:45,381] INFO : Plotted jet ratio +rank0 [2026-11-15 02:26:11,917] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 02:26:11,917] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:27:11,174] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 02:27:11,174] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:27:31,631] INFO : Step 3000/500000 rank0 | Train Loss: 3.4370 | Train Loss Unweighted: 1.3986 | Train Loss Components: Classification: 0.0044 | Classification_binary: 1.2261 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0833 | Regression_eta: 0.0004 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.3269 | Classification_binary: 1.2261 | Regression_cos_phi: 0.1289 | Regression_energy: 0.6615 | Regression_eta: 0.1409 | Regression_pt: 0.7026 | Regression_sin_phi: 0.2501 | LR: 2.05e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4460s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0579 | max_tensorboard_logging: 0.1169 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2754.1 | uss_MB: 1525.9 | child_rss_MB: 22746.2 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:27:31,633] INFO : Running validation on rank0 +rank0 [2026-11-15 02:27:31,633] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:27:31,633] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:27:31,750] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 02:27:31,855] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:27:31,955] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:27:32,022] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:27:32,089] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:27:32,155] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:27:32,221] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:27:32,289] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:27:32,397] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:27:32,499] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:27:32,601] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:27:32,668] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:27:32,733] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:27:32,796] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:27:32,861] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:27:32,925] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:27:33,038] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:27:33,044] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:27:33,046] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:27:33,049] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:27:33,049] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:27:33,053] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:27:33,053] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:27:33,055] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:27:33,057] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:27:33,059] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:27:33,061] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:27:33,063] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:27:33,064] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:27:33,064] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:27:33,068] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:27:33,070] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:27:34,529] INFO : VALIDATION | Step=3000/500000 | Train Loss=3.4370 | Valid Loss=5.0607 | Train Loss Unweighted=1.3986 | Valid Loss Unweighted=1.2039 | Stale=0 | ETA=954.2m +rank0 [2026-11-15 02:27:34,530] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:27:34,600] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:27:34,607] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:27:34,651] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:27:35,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:27:35,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:27:35,578] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:27:35,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:27:36,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:27:36,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:27:36,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:27:36,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:27:37,029] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:27:37,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:27:37,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:27:37,763] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:27:38,003] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:27:38,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:27:38,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:27:38,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:27:38,735] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:27:38,741] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:27:39,278] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:27:39,627] INFO : Plotted number of elements +rank0 [2026-11-15 02:27:40,855] INFO : Plotted jets +rank0 [2026-11-15 02:27:42,186] INFO : Plotted jet ratio +rank0 [2026-11-15 02:28:22,050] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 02:28:22,050] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:29:21,702] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 02:29:21,702] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:29:29,123] INFO : Step 4000/500000 rank0 | Train Loss: 4.8232 | Train Loss Unweighted: 1.0335 | Train Loss Components: Classification: 0.0021 | Classification_binary: 0.8629 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0807 | Regression_eta: 0.0045 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.1572 | Classification_binary: 0.8629 | Regression_cos_phi: 0.4461 | Regression_energy: 0.6408 | Regression_eta: 1.6570 | Regression_pt: 0.6889 | Regression_sin_phi: 0.3704 | LR: 2.08e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4716s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0404 | max_loss: 0.0028 | max_backward: 0.0400 | max_optimizer_step: 0.0598 | max_tensorboard_logging: 0.1271 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2787.9 | uss_MB: 1559.7 | child_rss_MB: 22744.9 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:29:29,125] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 02:29:29,191] INFO : Running validation on rank0 +rank0 [2026-11-15 02:29:29,191] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:29:29,191] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:29:29,310] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 02:29:29,414] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:29:29,514] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:29:29,581] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:29:29,646] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:29:29,709] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:29:29,775] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:29:29,841] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:29:29,947] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:29:30,049] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:29:30,153] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:29:30,220] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:29:30,285] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:29:30,350] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:29:30,417] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:29:30,483] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:29:30,579] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:29:30,584] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:29:30,585] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:29:30,589] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:29:30,589] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:29:30,591] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:29:30,593] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:29:30,595] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:29:30,597] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:29:30,599] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:29:30,601] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:29:30,603] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:29:30,603] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:29:30,605] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:29:30,607] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:29:30,609] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:29:32,073] INFO : VALIDATION | Step=4000/500000 | Train Loss=4.8232 | Valid Loss=4.9634 | Train Loss Unweighted=1.0335 | Valid Loss Unweighted=1.1719 | Stale=0 | ETA=957.1m +rank0 [2026-11-15 02:29:32,074] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:29:32,146] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:29:32,152] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:29:32,196] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:29:32,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:29:32,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:29:33,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:29:33,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:29:33,600] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:29:33,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:29:34,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:29:34,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:29:34,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:29:34,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:29:35,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:29:35,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:29:35,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:29:35,765] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:29:35,999] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:29:36,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:29:36,268] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:29:36,274] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:29:36,808] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:29:37,167] INFO : Plotted number of elements +rank0 [2026-11-15 02:29:38,401] INFO : Plotted jets +rank0 [2026-11-15 02:29:39,739] INFO : Plotted jet ratio +rank0 [2026-11-15 02:30:32,992] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 02:30:32,992] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:31:27,154] INFO : Step 5000/500000 rank0 | Train Loss: 2.7700 | Train Loss Unweighted: 0.9656 | Train Loss Components: Classification: 0.0065 | Classification_binary: 0.8477 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0559 | Regression_eta: 0.0004 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.4895 | Classification_binary: 0.8477 | Regression_cos_phi: 0.1431 | Regression_energy: 0.4445 | Regression_eta: 0.1579 | Regression_pt: 0.4556 | Regression_sin_phi: 0.2317 | LR: 2.13e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4524s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0582 | max_tensorboard_logging: 0.1227 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2811.2 | uss_MB: 1583.0 | child_rss_MB: 22744.5 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:31:27,156] INFO : Running validation on rank0 +rank0 [2026-11-15 02:31:27,156] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:31:27,156] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:31:27,282] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 02:31:27,388] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:31:27,490] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:31:27,559] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:31:27,626] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:31:27,692] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:31:27,758] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:31:27,823] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:31:27,930] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:31:28,031] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:31:28,132] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:31:28,199] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:31:28,262] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:31:28,325] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:31:28,390] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:31:28,454] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:31:28,565] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:31:28,572] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:31:28,574] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:31:28,576] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:31:28,576] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:31:28,580] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:31:28,580] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:31:28,582] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:31:28,584] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:31:28,585] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:31:28,588] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:31:28,590] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:31:28,590] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:31:28,592] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:31:28,594] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:31:28,595] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:31:30,061] INFO : VALIDATION | Step=5000/500000 | Train Loss=2.7700 | Valid Loss=4.9318 | Train Loss Unweighted=0.9656 | Valid Loss Unweighted=1.1745 | Stale=0 | ETA=958.8m +rank0 [2026-11-15 02:31:30,062] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:31:30,134] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:31:30,140] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:31:30,180] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:31:30,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:31:30,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:31:31,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:31:31,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:31:31,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:31:31,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:31:32,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:31:32,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:31:32,558] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:31:32,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:31:33,027] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:31:33,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:31:33,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:31:33,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:31:33,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:31:34,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:31:34,255] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:31:34,260] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:31:34,787] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:31:35,128] INFO : Plotted number of elements +rank0 [2026-11-15 02:31:36,359] INFO : Plotted jets +rank0 [2026-11-15 02:31:37,668] INFO : Plotted jet ratio +rank0 [2026-11-15 02:31:44,117] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 02:31:44,117] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:32:44,187] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 02:32:44,187] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:33:25,142] INFO : Step 6000/500000 rank0 | Train Loss: 6.7354 | Train Loss Unweighted: 1.4612 | Train Loss Components: Classification: 0.0093 | Classification_binary: 1.2931 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0796 | Regression_eta: 0.0013 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.6920 | Classification_binary: 1.2931 | Regression_cos_phi: 0.4387 | Regression_energy: 0.6327 | Regression_eta: 0.4831 | Regression_pt: 0.6247 | Regression_sin_phi: 2.5710 | LR: 2.19e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4709s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0440 | max_optimizer_step: 0.0621 | max_tensorboard_logging: 0.1281 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2765.7 | uss_MB: 1537.5 | child_rss_MB: 22745.0 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:33:25,144] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 02:33:25,210] INFO : Running validation on rank0 +rank0 [2026-11-15 02:33:25,210] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:33:25,210] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:33:25,328] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 02:33:25,434] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:33:25,534] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:33:25,602] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:33:25,666] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:33:25,730] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:33:25,796] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:33:25,863] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:33:25,971] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:33:26,073] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:33:26,174] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:33:26,241] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:33:26,308] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:33:26,373] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:33:26,440] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:33:26,506] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:33:26,601] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:33:26,606] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:33:26,609] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:33:26,611] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:33:26,612] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:33:26,614] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:33:26,616] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:33:26,617] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:33:26,620] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:33:26,620] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:33:26,622] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:33:26,625] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:33:26,626] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:33:26,628] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:33:26,630] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:33:26,631] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:33:28,093] INFO : VALIDATION | Step=6000/500000 | Train Loss=6.7354 | Valid Loss=4.9036 | Train Loss Unweighted=1.4612 | Valid Loss Unweighted=1.1576 | Stale=0 | ETA=959.4m +rank0 [2026-11-15 02:33:28,093] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:33:28,163] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:33:28,170] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:33:28,214] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:33:28,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:33:28,890] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:33:29,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:33:29,377] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:33:29,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:33:29,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:33:30,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:33:30,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:33:30,562] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:33:30,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:33:31,047] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:33:31,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:33:31,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:33:31,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:33:32,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:33:32,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:33:32,273] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:33:32,279] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:33:32,815] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:33:33,173] INFO : Plotted number of elements +rank0 [2026-11-15 02:33:34,404] INFO : Plotted jets +rank0 [2026-11-15 02:33:35,735] INFO : Plotted jet ratio +rank0 [2026-11-15 02:33:55,451] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 02:33:55,451] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:34:55,278] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 02:34:55,278] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:35:22,923] INFO : Step 7000/500000 rank0 | Train Loss: 2.1596 | Train Loss Unweighted: 0.7509 | Train Loss Components: Classification: 0.0008 | Classification_binary: 0.6251 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0626 | Regression_eta: 0.0001 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.6251 | Regression_cos_phi: 0.2742 | Regression_energy: 0.4973 | Regression_eta: 0.0497 | Regression_pt: 0.5155 | Regression_sin_phi: 0.1374 | LR: 2.26e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4530s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0586 | max_tensorboard_logging: 0.1228 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2863.7 | uss_MB: 1635.5 | child_rss_MB: 22744.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:35:22,925] INFO : Running validation on rank0 +rank0 [2026-11-15 02:35:22,925] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:35:22,925] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:35:23,043] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 02:35:23,148] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:35:23,250] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:35:23,320] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:35:23,387] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:35:23,453] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:35:23,520] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:35:23,587] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:35:23,696] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:35:23,795] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:35:23,893] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:35:23,958] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:35:24,023] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:35:24,088] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:35:24,153] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:35:24,218] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:35:24,313] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:35:24,318] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:35:24,319] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:35:24,323] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:35:24,325] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:35:24,327] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:35:24,327] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:35:24,329] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:35:24,331] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:35:24,333] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:35:24,335] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:35:24,337] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:35:24,338] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:35:24,339] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:35:24,340] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:35:24,343] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:35:25,747] INFO : VALIDATION | Step=7000/500000 | Train Loss=2.1596 | Valid Loss=4.9072 | Train Loss Unweighted=0.7509 | Valid Loss Unweighted=1.1484 | Stale=1 | ETA=958.8m +rank0 [2026-11-15 02:35:25,747] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:35:25,820] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:35:25,828] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:35:25,872] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:35:26,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:35:26,600] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:35:26,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:35:27,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:35:27,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:35:27,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:35:27,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:35:28,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:35:28,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:35:28,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:35:28,766] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:35:29,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:35:29,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:35:29,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:35:29,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:35:29,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:35:29,971] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:35:29,976] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:35:30,511] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:35:30,858] INFO : Plotted number of elements +rank0 [2026-11-15 02:35:32,100] INFO : Plotted jets +rank0 [2026-11-15 02:35:33,439] INFO : Plotted jet ratio +rank0 [2026-11-15 02:36:06,291] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 02:36:06,291] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:37:06,192] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 02:37:06,192] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:37:20,639] INFO : Step 8000/500000 rank0 | Train Loss: 6.1202 | Train Loss Unweighted: 0.6884 | Train Loss Components: Classification: 0.0037 | Classification_binary: 0.5405 | Regression_cos_phi: 0.0036 | Regression_energy: 0.0715 | Regression_eta: 0.0006 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.2783 | Classification_binary: 0.5405 | Regression_cos_phi: 2.9209 | Regression_energy: 0.5680 | Regression_eta: 0.2105 | Regression_pt: 0.5626 | Regression_sin_phi: 1.0393 | LR: 2.34e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4758s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0797 | max_tensorboard_logging: 0.1214 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2886.4 | uss_MB: 1658.2 | child_rss_MB: 22742.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:37:20,641] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 02:37:20,707] INFO : Running validation on rank0 +rank0 [2026-11-15 02:37:20,707] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:37:20,707] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:37:20,824] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 02:37:20,929] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:37:21,030] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:37:21,099] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:37:21,163] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:37:21,227] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:37:21,292] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:37:21,357] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:37:21,466] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:37:21,566] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:37:21,666] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:37:21,734] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:37:21,800] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:37:21,865] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:37:21,931] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:37:21,997] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:37:22,096] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:37:22,100] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:37:22,103] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:37:22,106] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:37:22,109] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:37:22,109] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:37:22,111] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:37:22,113] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:37:22,115] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:37:22,117] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:37:22,118] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:37:22,121] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:37:22,122] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:37:22,123] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:37:22,125] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:37:22,128] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:37:23,519] INFO : VALIDATION | Step=8000/500000 | Train Loss=6.1202 | Valid Loss=4.9302 | Train Loss Unweighted=0.6884 | Valid Loss Unweighted=1.1449 | Stale=2 | ETA=957.9m +rank0 [2026-11-15 02:37:23,520] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:37:23,589] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:37:23,596] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:37:23,642] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:37:24,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:37:24,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:37:24,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:37:24,831] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:37:25,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:37:25,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:37:25,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:37:25,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:37:26,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:37:26,253] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:37:26,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:37:26,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:37:26,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:37:27,217] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:37:27,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:37:27,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:37:27,715] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:37:27,720] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:37:28,256] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:37:28,610] INFO : Plotted number of elements +rank0 [2026-11-15 02:37:29,831] INFO : Plotted jets +rank0 [2026-11-15 02:37:31,165] INFO : Plotted jet ratio +rank0 [2026-11-15 02:38:17,228] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 02:38:17,229] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:39:16,917] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 02:39:16,917] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:39:18,146] INFO : Step 9000/500000 rank0 | Train Loss: 2.1259 | Train Loss Unweighted: 0.7982 | Train Loss Components: Classification: 0.0014 | Classification_binary: 0.7323 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0327 | Regression_eta: 0.0002 | Regression_pt: 0.0308 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.1013 | Classification_binary: 0.7323 | Regression_cos_phi: 0.3060 | Regression_energy: 0.2599 | Regression_eta: 0.0821 | Regression_pt: 0.2574 | Regression_sin_phi: 0.3871 | LR: 2.43e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4641s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0591 | max_tensorboard_logging: 0.1271 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2839.4 | uss_MB: 1611.2 | child_rss_MB: 22735.3 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:39:18,147] INFO : Running validation on rank0 +rank0 [2026-11-15 02:39:18,147] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:39:18,147] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:39:18,265] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 02:39:18,371] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:39:18,471] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:39:18,540] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:39:18,607] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:39:18,673] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:39:18,740] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:39:18,806] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:39:18,917] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:39:19,021] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:39:19,125] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:39:19,191] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:39:19,255] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:39:19,317] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:39:19,381] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:39:19,446] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:39:19,541] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:39:19,547] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:39:19,549] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:39:19,550] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:39:19,553] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:39:19,555] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:39:19,556] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:39:19,558] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:39:19,559] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:39:19,560] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:39:19,563] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:39:19,565] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:39:19,566] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:39:19,568] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:39:19,570] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:39:19,571] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:39:20,961] INFO : VALIDATION | Step=9000/500000 | Train Loss=2.1259 | Valid Loss=4.9481 | Train Loss Unweighted=0.7982 | Valid Loss Unweighted=1.1884 | Stale=3 | ETA=956.5m +rank0 [2026-11-15 02:39:20,961] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:39:21,032] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:39:21,038] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:39:21,084] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:39:21,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:39:21,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:39:22,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:39:22,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:39:22,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:39:22,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:39:22,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:39:23,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:39:23,450] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:39:23,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:39:23,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:39:24,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:39:24,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:39:24,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:39:24,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:39:25,110] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:39:25,160] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:39:25,166] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:39:25,705] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:39:26,067] INFO : Plotted number of elements +rank0 [2026-11-15 02:39:27,298] INFO : Plotted jets +rank0 [2026-11-15 02:39:28,622] INFO : Plotted jet ratio +rank0 [2026-11-15 02:40:27,929] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 02:40:27,929] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:41:15,683] INFO : Step 10000/500000 rank0 | Train Loss: 2.8813 | Train Loss Unweighted: 0.8393 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.7836 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0259 | Regression_eta: 0.0032 | Regression_pt: 0.0256 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.7836 | Regression_cos_phi: 0.1517 | Regression_energy: 0.2056 | Regression_eta: 1.1859 | Regression_pt: 0.2140 | Regression_sin_phi: 0.3075 | LR: 2.52e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4552s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0583 | max_tensorboard_logging: 0.1246 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2935.5 | uss_MB: 1707.3 | child_rss_MB: 22744.2 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:41:15,685] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 02:41:15,749] INFO : Running validation on rank0 +rank0 [2026-11-15 02:41:15,749] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:41:15,749] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:41:15,865] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 02:41:15,969] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:41:16,072] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:41:16,141] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:41:16,206] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:41:16,272] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:41:16,340] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:41:16,407] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:41:16,520] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:41:16,624] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:41:16,726] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:41:16,792] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:41:16,856] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:41:16,920] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:41:16,985] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:41:17,050] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:41:17,149] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:41:17,155] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:41:17,155] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:41:17,157] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:41:17,159] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:41:17,163] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:41:17,163] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:41:17,165] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:41:17,167] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:41:17,168] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:41:17,171] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:41:17,173] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:41:17,173] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:41:17,174] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:41:17,177] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:41:17,179] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:41:18,571] INFO : VALIDATION | Step=10000/500000 | Train Loss=2.8813 | Valid Loss=5.0090 | Train Loss Unweighted=0.8393 | Valid Loss Unweighted=1.1788 | Stale=4 | ETA=955.2m +rank0 [2026-11-15 02:41:18,572] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:41:18,646] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:41:18,654] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:41:18,700] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:41:19,161] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:41:19,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:41:19,655] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:41:19,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:41:20,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:41:20,373] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:41:20,609] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:41:20,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:41:21,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:41:21,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:41:21,574] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:41:21,827] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:41:22,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:41:22,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:41:22,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:41:22,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:41:22,813] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:41:22,818] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:41:23,357] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:41:23,705] INFO : Plotted number of elements +rank0 [2026-11-15 02:41:24,947] INFO : Plotted jets +rank0 [2026-11-15 02:41:26,297] INFO : Plotted jet ratio +rank0 [2026-11-15 02:41:39,015] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 02:41:39,015] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:42:39,151] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 02:42:39,151] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:43:13,968] INFO : Step 11000/500000 rank0 | Train Loss: 2.7568 | Train Loss Unweighted: 1.3722 | Train Loss Components: Classification: 0.0038 | Classification_binary: 1.3126 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0268 | Regression_eta: 0.0005 | Regression_pt: 0.0279 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.2805 | Classification_binary: 1.3126 | Regression_cos_phi: 0.3194 | Regression_energy: 0.2130 | Regression_eta: 0.1771 | Regression_pt: 0.2334 | Regression_sin_phi: 0.2209 | LR: 2.63e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4863s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0386 | max_optimizer_step: 0.0802 | max_tensorboard_logging: 0.1281 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2962.7 | uss_MB: 1734.5 | child_rss_MB: 22746.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:43:13,970] INFO : Running validation on rank0 +rank0 [2026-11-15 02:43:13,970] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:43:13,970] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:43:14,089] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 02:43:14,193] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:43:14,292] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:43:14,360] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:43:14,426] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:43:14,490] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:43:14,556] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:43:14,621] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:43:14,728] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:43:14,828] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:43:14,927] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:43:14,994] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:43:15,061] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:43:15,127] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:43:15,193] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:43:15,258] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:43:15,355] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:43:15,361] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:43:15,363] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:43:15,365] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:43:15,367] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:43:15,369] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:43:15,370] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:43:15,372] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:43:15,372] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:43:15,374] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:43:15,376] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:43:15,379] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:43:15,380] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:43:15,381] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:43:15,382] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:43:15,386] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:43:16,851] INFO : VALIDATION | Step=11000/500000 | Train Loss=2.7568 | Valid Loss=4.8994 | Train Loss Unweighted=1.3722 | Valid Loss Unweighted=1.1361 | Stale=0 | ETA=954.2m +rank0 [2026-11-15 02:43:16,853] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:43:16,928] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:43:16,935] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:43:16,981] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:43:17,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:43:17,684] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:43:17,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:43:18,176] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:43:18,417] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:43:18,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:43:18,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:43:19,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:43:19,368] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:43:19,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:43:19,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:43:20,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:43:20,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:43:20,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:43:20,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:43:21,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:43:21,062] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:43:21,079] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:43:21,613] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:43:21,966] INFO : Plotted number of elements +rank0 [2026-11-15 02:43:23,207] INFO : Plotted jets +rank0 [2026-11-15 02:43:24,550] INFO : Plotted jet ratio +rank0 [2026-11-15 02:43:50,523] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 02:43:50,523] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:44:50,533] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 02:44:50,533] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:45:11,975] INFO : Step 12000/500000 rank0 | Train Loss: 1.0232 | Train Loss Unweighted: 0.4469 | Train Loss Components: Classification: 0.0013 | Classification_binary: 0.4117 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0168 | Regression_eta: 0.0002 | Regression_pt: 0.0167 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0959 | Classification_binary: 0.4117 | Regression_cos_phi: 0.1092 | Regression_energy: 0.1337 | Regression_eta: 0.0621 | Regression_pt: 0.1391 | Regression_sin_phi: 0.0714 | LR: 2.75e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4685s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0357 | max_loss: 0.0026 | max_backward: 0.0404 | max_optimizer_step: 0.0598 | max_tensorboard_logging: 0.1246 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2983.6 | uss_MB: 1755.4 | child_rss_MB: 22746.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:45:11,976] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 02:45:12,044] INFO : Running validation on rank0 +rank0 [2026-11-15 02:45:12,044] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:45:12,044] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:45:12,162] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 02:45:12,267] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:45:12,368] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:45:12,435] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:45:12,501] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:45:12,565] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:45:12,631] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:45:12,696] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:45:12,804] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:45:12,905] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:45:13,007] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:45:13,076] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:45:13,143] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:45:13,209] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:45:13,275] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:45:13,341] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:45:13,438] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:45:13,444] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:45:13,446] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:45:13,448] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:45:13,448] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:45:13,452] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:45:13,452] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:45:13,454] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:45:13,456] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:45:13,458] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:45:13,460] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:45:13,462] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:45:13,463] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:45:13,463] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:45:13,466] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:45:13,467] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:45:14,858] INFO : VALIDATION | Step=12000/500000 | Train Loss=1.0232 | Valid Loss=5.2057 | Train Loss Unweighted=0.4469 | Valid Loss Unweighted=1.2596 | Stale=1 | ETA=952.9m +rank0 [2026-11-15 02:45:14,858] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:45:14,928] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:45:14,935] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:45:14,982] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:45:15,438] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:45:15,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:45:15,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:45:16,173] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:45:16,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:45:16,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:45:16,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:45:17,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:45:17,393] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:45:17,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:45:17,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:45:18,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:45:18,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:45:18,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:45:18,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:45:19,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:45:19,124] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:45:19,130] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:45:19,667] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:45:20,023] INFO : Plotted number of elements +rank0 [2026-11-15 02:45:21,253] INFO : Plotted jets +rank0 [2026-11-15 02:45:22,586] INFO : Plotted jet ratio +rank0 [2026-11-15 02:46:01,778] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 02:46:01,778] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:47:01,584] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 02:47:01,584] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:47:09,812] INFO : Step 13000/500000 rank0 | Train Loss: 2.7919 | Train Loss Unweighted: 0.9411 | Train Loss Components: Classification: 0.0025 | Classification_binary: 0.7659 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0833 | Regression_eta: 0.0005 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.1856 | Classification_binary: 0.7659 | Regression_cos_phi: 0.1062 | Regression_energy: 0.6622 | Regression_eta: 0.1931 | Regression_pt: 0.7399 | Regression_sin_phi: 0.1391 | LR: 2.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4604s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0028 | max_backward: 0.0391 | max_optimizer_step: 0.0580 | max_tensorboard_logging: 0.1248 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3016.7 | uss_MB: 1788.5 | child_rss_MB: 22746.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:47:09,814] INFO : Running validation on rank0 +rank0 [2026-11-15 02:47:09,814] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:47:09,814] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:47:09,932] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 02:47:10,037] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:47:10,138] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:47:10,207] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:47:10,274] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:47:10,340] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:47:10,407] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:47:10,474] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:47:10,585] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:47:10,686] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:47:10,784] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:47:10,850] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:47:10,915] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:47:10,978] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:47:11,043] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:47:11,109] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:47:11,204] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:47:11,210] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:47:11,210] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:47:11,213] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:47:11,216] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:47:11,216] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:47:11,218] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:47:11,220] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:47:11,222] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:47:11,224] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:47:11,226] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:47:11,228] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:47:11,229] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:47:11,230] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:47:11,232] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:47:11,235] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:47:12,623] INFO : VALIDATION | Step=13000/500000 | Train Loss=2.7919 | Valid Loss=5.2815 | Train Loss Unweighted=0.9411 | Valid Loss Unweighted=1.2289 | Stale=2 | ETA=951.3m +rank0 [2026-11-15 02:47:12,623] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:47:12,696] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:47:12,702] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:47:12,748] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:47:13,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:47:13,457] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:47:13,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:47:13,947] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:47:14,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:47:14,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:47:14,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:47:14,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:47:15,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:47:15,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:47:15,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:47:15,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:47:16,074] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:47:16,304] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:47:16,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:47:16,755] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:47:16,808] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:47:16,813] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:47:17,346] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:47:17,684] INFO : Plotted number of elements +rank0 [2026-11-15 02:47:18,907] INFO : Plotted jets +rank0 [2026-11-15 02:47:20,226] INFO : Plotted jet ratio +rank0 [2026-11-15 02:48:11,857] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 02:48:11,857] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:49:06,018] INFO : Step 14000/500000 rank0 | Train Loss: 2.8806 | Train Loss Unweighted: 0.9824 | Train Loss Components: Classification: 0.0018 | Classification_binary: 0.8855 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0469 | Regression_eta: 0.0015 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.1358 | Classification_binary: 0.8855 | Regression_cos_phi: 0.2221 | Regression_energy: 0.3729 | Regression_eta: 0.5385 | Regression_pt: 0.3845 | Regression_sin_phi: 0.3413 | LR: 3.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4647s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0595 | max_tensorboard_logging: 0.1283 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3032.1 | uss_MB: 1803.9 | child_rss_MB: 22746.4 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:49:06,023] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 02:49:06,089] INFO : Running validation on rank0 +rank0 [2026-11-15 02:49:06,089] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:49:06,089] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:49:06,205] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 02:49:06,312] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:49:06,414] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:49:06,482] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:49:06,548] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:49:06,615] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:49:06,682] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:49:06,750] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:49:06,861] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:49:06,962] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:49:07,063] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:49:07,131] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:49:07,195] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:49:07,259] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:49:07,323] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:49:07,387] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:49:07,483] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:49:07,489] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:49:07,491] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:49:07,492] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:49:07,495] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:49:07,495] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:49:07,497] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:49:07,499] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:49:07,501] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:49:07,503] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:49:07,505] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:49:07,507] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:49:07,508] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:49:07,509] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:49:07,511] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:49:07,514] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:49:08,895] INFO : VALIDATION | Step=14000/500000 | Train Loss=2.8806 | Valid Loss=5.3833 | Train Loss Unweighted=0.9824 | Valid Loss Unweighted=1.2278 | Stale=3 | ETA=948.8m +rank0 [2026-11-15 02:49:08,895] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:49:08,968] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:49:08,975] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:49:09,021] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:49:09,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:49:09,752] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:49:09,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:49:10,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:49:10,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:49:10,716] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:49:10,953] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:49:11,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:49:11,453] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:49:11,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:49:11,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:49:12,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:49:12,421] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:49:12,659] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:49:12,899] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:49:13,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:49:13,170] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:49:13,175] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:49:13,708] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:49:14,062] INFO : Plotted number of elements +rank0 [2026-11-15 02:49:15,299] INFO : Plotted jets +rank0 [2026-11-15 02:49:16,634] INFO : Plotted jet ratio +rank0 [2026-11-15 02:49:22,270] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 02:49:22,270] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:50:21,811] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 02:50:21,811] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:51:03,307] INFO : Step 15000/500000 rank0 | Train Loss: 2.7781 | Train Loss Unweighted: 1.2876 | Train Loss Components: Classification: 0.0044 | Classification_binary: 1.2138 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0292 | Regression_eta: 0.0007 | Regression_pt: 0.0392 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.3266 | Classification_binary: 1.2138 | Regression_cos_phi: 0.1044 | Regression_energy: 0.2319 | Regression_eta: 0.2555 | Regression_pt: 0.3272 | Regression_sin_phi: 0.3187 | LR: 3.17e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4656s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0394 | max_optimizer_step: 0.0626 | max_tensorboard_logging: 0.1207 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3058.7 | uss_MB: 1830.5 | child_rss_MB: 22746.2 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:51:03,308] INFO : Running validation on rank0 +rank0 [2026-11-15 02:51:03,308] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:51:03,308] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:51:03,425] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 02:51:03,531] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:51:03,631] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:51:03,700] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:51:03,766] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:51:03,834] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:51:03,903] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:51:03,970] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:51:04,082] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:51:04,185] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:51:04,285] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:51:04,352] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:51:04,416] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:51:04,480] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:51:04,544] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:51:04,608] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:51:04,706] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:51:04,712] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:51:04,714] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:51:04,715] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:51:04,718] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:51:04,720] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:51:04,721] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:51:04,723] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:51:04,724] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:51:04,727] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:51:04,728] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:51:04,730] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:51:04,731] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:51:04,733] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:51:04,734] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:51:04,737] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:51:06,133] INFO : VALIDATION | Step=15000/500000 | Train Loss=2.7781 | Valid Loss=5.4673 | Train Loss Unweighted=1.2876 | Valid Loss Unweighted=1.2550 | Stale=4 | ETA=946.9m +rank0 [2026-11-15 02:51:06,133] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:51:06,205] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:51:06,211] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:51:06,258] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:51:06,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:51:06,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:51:07,221] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:51:07,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:51:07,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:51:07,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:51:08,178] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:51:08,415] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:51:08,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:51:08,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:51:09,145] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:51:09,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:51:09,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:51:09,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:51:10,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:51:10,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:51:10,362] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:51:10,367] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:51:10,903] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:51:11,250] INFO : Plotted number of elements +rank0 [2026-11-15 02:51:12,485] INFO : Plotted jets +rank0 [2026-11-15 02:51:13,840] INFO : Plotted jet ratio +rank0 [2026-11-15 02:51:32,739] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 02:51:32,739] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:52:32,556] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 02:52:32,556] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:53:00,864] INFO : Step 16000/500000 rank0 | Train Loss: 2.0774 | Train Loss Unweighted: 0.5840 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.5179 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0316 | Regression_eta: 0.0004 | Regression_pt: 0.0327 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0332 | Classification_binary: 0.5179 | Regression_cos_phi: 0.5938 | Regression_energy: 0.2512 | Regression_eta: 0.1317 | Regression_pt: 0.2729 | Regression_sin_phi: 0.2768 | LR: 3.33e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4663s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0387 | max_optimizer_step: 0.0609 | max_tensorboard_logging: 0.1224 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3088.9 | uss_MB: 1860.7 | child_rss_MB: 22743.7 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:53:00,866] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 02:53:00,933] INFO : Running validation on rank0 +rank0 [2026-11-15 02:53:00,933] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:53:00,933] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:53:01,052] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 02:53:01,159] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:53:01,261] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:53:01,330] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:53:01,396] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:53:01,462] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:53:01,531] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:53:01,599] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:53:01,710] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:53:01,813] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:53:01,912] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:53:01,979] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:53:02,046] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:53:02,112] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:53:02,177] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:53:02,242] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:53:02,340] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:53:02,346] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:53:02,348] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:53:02,350] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:53:02,352] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:53:02,354] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:53:02,355] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:53:02,356] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:53:02,357] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:53:02,360] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:53:02,363] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:53:02,365] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:53:02,365] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:53:02,367] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:53:02,369] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:53:02,371] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:53:03,760] INFO : VALIDATION | Step=16000/500000 | Train Loss=2.0774 | Valid Loss=5.4978 | Train Loss Unweighted=0.5840 | Valid Loss Unweighted=1.3049 | Stale=5 | ETA=945.2m +rank0 [2026-11-15 02:53:03,760] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:53:03,832] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:53:03,839] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:53:03,886] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:53:04,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:53:04,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:53:04,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:53:05,113] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:53:05,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:53:05,584] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:53:05,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:53:06,070] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:53:06,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:53:06,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:53:06,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:53:07,027] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:53:07,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:53:07,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:53:07,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:53:07,960] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:53:08,006] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:53:08,012] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:53:08,546] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:53:08,890] INFO : Plotted number of elements +rank0 [2026-11-15 02:53:10,131] INFO : Plotted jets +rank0 [2026-11-15 02:53:11,517] INFO : Plotted jet ratio +rank0 [2026-11-15 02:53:43,504] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 02:53:43,504] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:54:42,645] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 02:54:42,645] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:54:57,862] INFO : Step 17000/500000 rank0 | Train Loss: 1.1839 | Train Loss Unweighted: 0.3385 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.2588 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0396 | Regression_eta: 0.0001 | Regression_pt: 0.0395 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0153 | Classification_binary: 0.2588 | Regression_cos_phi: 0.0858 | Regression_energy: 0.3143 | Regression_eta: 0.0517 | Regression_pt: 0.3300 | Regression_sin_phi: 0.1280 | LR: 3.51e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4707s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0602 | max_tensorboard_logging: 0.1243 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3105.0 | uss_MB: 1876.8 | child_rss_MB: 22739.1 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:54:57,863] INFO : Running validation on rank0 +rank0 [2026-11-15 02:54:57,863] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:54:57,863] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:54:57,981] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 02:54:58,090] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:54:58,193] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:54:58,260] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:54:58,327] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:54:58,393] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:54:58,460] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:54:58,528] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:54:58,637] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:54:58,737] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:54:58,837] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:54:58,903] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:54:58,967] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:54:59,032] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:54:59,099] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:54:59,163] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:54:59,259] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:54:59,265] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:54:59,268] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:54:59,268] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:54:59,270] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:54:59,273] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:54:59,274] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:54:59,276] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:54:59,278] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:54:59,280] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:54:59,281] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:54:59,282] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:54:59,284] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:54:59,286] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:54:59,288] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:54:59,290] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:55:00,662] INFO : VALIDATION | Step=17000/500000 | Train Loss=1.1839 | Valid Loss=5.5486 | Train Loss Unweighted=0.3385 | Valid Loss Unweighted=1.3199 | Stale=6 | ETA=943.1m +rank0 [2026-11-15 02:55:00,663] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:55:00,735] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:55:00,741] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:55:00,786] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:55:01,254] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:55:01,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:55:01,741] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:55:01,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:55:02,221] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:55:02,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:55:02,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:55:02,932] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:55:03,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:55:03,418] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:55:03,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:55:03,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:55:04,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:55:04,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:55:04,603] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:55:04,819] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:55:04,875] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:55:04,881] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:55:05,417] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:55:05,760] INFO : Plotted number of elements +rank0 [2026-11-15 02:55:06,994] INFO : Plotted jets +rank0 [2026-11-15 02:55:08,346] INFO : Plotted jet ratio +rank0 [2026-11-15 02:55:53,478] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 02:55:53,478] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:56:52,748] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 02:56:52,748] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:56:54,824] INFO : Step 18000/500000 rank0 | Train Loss: 1.5685 | Train Loss Unweighted: 0.5531 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.4800 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0361 | Regression_eta: 0.0003 | Regression_pt: 0.0360 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0145 | Classification_binary: 0.4800 | Regression_cos_phi: 0.0981 | Regression_energy: 0.2869 | Regression_eta: 0.1241 | Regression_pt: 0.3010 | Regression_sin_phi: 0.2639 | LR: 3.69e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4622s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0388 | max_optimizer_step: 0.0602 | max_tensorboard_logging: 0.1208 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3130.2 | uss_MB: 1902.0 | child_rss_MB: 22745.1 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:56:54,826] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 02:56:54,889] INFO : Running validation on rank0 +rank0 [2026-11-15 02:56:54,889] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:56:54,889] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:56:55,018] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 02:56:55,125] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:56:55,228] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:56:55,297] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:56:55,363] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:56:55,429] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:56:55,497] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:56:55,566] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:56:55,676] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:56:55,778] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:56:55,879] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:56:55,945] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:56:56,011] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:56:56,077] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:56:56,142] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:56:56,207] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:56:56,305] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:56:56,310] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:56:56,313] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:56:56,316] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:56:56,316] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:56:56,319] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:56:56,320] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:56:56,322] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:56:56,324] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:56:56,326] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:56:56,327] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:56:56,330] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:56:56,330] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:56:56,332] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:56:56,334] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:56:56,335] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:56:57,730] INFO : VALIDATION | Step=18000/500000 | Train Loss=1.5685 | Valid Loss=5.8648 | Train Loss Unweighted=0.5531 | Valid Loss Unweighted=1.3450 | Stale=7 | ETA=941.1m +rank0 [2026-11-15 02:56:57,731] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:56:57,802] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:56:57,809] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:56:57,856] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:56:58,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:56:58,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:56:58,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:56:59,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:56:59,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:56:59,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:56:59,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:57:00,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:57:00,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:57:00,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:57:00,725] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:57:00,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:57:01,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:57:01,429] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:57:01,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:57:01,881] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:57:01,936] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:57:01,942] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:57:02,480] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:57:02,820] INFO : Plotted number of elements +rank0 [2026-11-15 02:57:04,047] INFO : Plotted jets +rank0 [2026-11-15 02:57:05,384] INFO : Plotted jet ratio +rank0 [2026-11-15 02:58:03,335] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 02:58:03,335] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:58:51,581] INFO : Step 19000/500000 rank0 | Train Loss: 1.9427 | Train Loss Unweighted: 0.7975 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.7566 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0198 | Regression_eta: 0.0002 | Regression_pt: 0.0195 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0320 | Classification_binary: 0.7566 | Regression_cos_phi: 0.5493 | Regression_energy: 0.1572 | Regression_eta: 0.0896 | Regression_pt: 0.1633 | Regression_sin_phi: 0.1947 | LR: 3.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4772s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0026 | max_backward: 0.0431 | max_optimizer_step: 0.0635 | max_tensorboard_logging: 0.1213 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3162.4 | uss_MB: 1934.3 | child_rss_MB: 22745.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:58:51,582] INFO : Running validation on rank0 +rank0 [2026-11-15 02:58:51,582] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:58:51,582] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:58:51,700] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 02:58:51,803] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:58:51,904] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:58:51,971] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:58:52,036] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:58:52,102] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:58:52,167] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:58:52,233] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:58:52,341] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:58:52,441] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:58:52,542] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:58:52,610] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:58:52,676] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:58:52,741] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:58:52,808] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:58:52,874] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:58:52,975] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:58:52,979] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:58:52,983] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:58:52,984] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:58:52,987] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:58:52,988] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:58:52,990] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:58:52,990] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:58:52,993] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:58:52,995] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:58:52,997] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:58:52,999] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:58:53,000] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:58:53,000] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:58:53,002] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:58:53,006] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:58:54,387] INFO : VALIDATION | Step=19000/500000 | Train Loss=1.9427 | Valid Loss=6.0428 | Train Loss Unweighted=0.7975 | Valid Loss Unweighted=1.4201 | Stale=8 | ETA=939.0m +rank0 [2026-11-15 02:58:54,388] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:58:54,457] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:58:54,463] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:58:54,509] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:58:54,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:58:55,198] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:58:55,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:58:55,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:58:55,919] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:58:56,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:58:56,404] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:58:56,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:58:56,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:58:57,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:58:57,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:58:57,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:58:57,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:58:58,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:58:58,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:58:58,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:58:58,572] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:58:58,579] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:58:59,103] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:58:59,454] INFO : Plotted number of elements +rank0 [2026-11-15 02:59:00,675] INFO : Plotted jets +rank0 [2026-11-15 02:59:02,004] INFO : Plotted jet ratio +rank0 [2026-11-15 02:59:13,692] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 02:59:13,692] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:00:12,894] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 03:00:12,895] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:00:48,053] INFO : Step 20000/500000 rank0 | Train Loss: 0.6223 | Train Loss Unweighted: 0.1651 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.1300 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0170 | Regression_eta: 0.0001 | Regression_pt: 0.0177 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0130 | Classification_binary: 0.1300 | Regression_cos_phi: 0.0804 | Regression_energy: 0.1348 | Regression_eta: 0.0442 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0722 | LR: 4.07e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4933s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0415 | max_optimizer_step: 0.0620 | max_tensorboard_logging: 0.1259 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3110.5 | uss_MB: 1882.3 | child_rss_MB: 22747.8 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:00:48,055] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 03:00:48,121] INFO : Running validation on rank0 +rank0 [2026-11-15 03:00:48,121] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:00:48,121] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:00:48,238] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 03:00:48,345] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:00:48,445] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:00:48,512] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:00:48,577] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:00:48,641] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:00:48,706] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:00:48,771] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:00:48,882] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:00:48,986] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:00:49,091] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:00:49,158] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:00:49,225] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:00:49,292] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:00:49,359] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:00:49,426] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:00:49,527] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:00:49,531] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:00:49,535] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:00:49,537] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:00:49,539] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:00:49,541] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:00:49,542] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:00:49,544] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:00:49,544] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:00:49,546] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:00:49,549] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:00:49,551] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:00:49,552] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:00:49,554] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:00:49,554] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:00:49,558] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:00:50,955] INFO : VALIDATION | Step=20000/500000 | Train Loss=0.6223 | Valid Loss=6.2160 | Train Loss Unweighted=0.1651 | Valid Loss Unweighted=1.4715 | Stale=9 | ETA=936.8m +rank0 [2026-11-15 03:00:50,956] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:00:51,027] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:00:51,034] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:00:51,080] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:00:51,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:00:51,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:00:52,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:00:52,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:00:52,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:00:52,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:00:52,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:00:53,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:00:53,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:00:53,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:00:53,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:00:54,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:00:54,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:00:54,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:00:54,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:00:55,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:00:55,179] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:00:55,184] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:00:55,715] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:00:56,071] INFO : Plotted number of elements +rank0 [2026-11-15 03:00:57,315] INFO : Plotted jets +rank0 [2026-11-15 03:00:58,656] INFO : Plotted jet ratio +rank0 [2026-11-15 03:01:23,636] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 03:01:23,636] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:02:23,263] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 03:02:23,264] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:02:45,480] INFO : Step 21000/500000 rank0 | Train Loss: 1.2808 | Train Loss Unweighted: 0.4894 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.4064 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0413 | Regression_eta: 0.0002 | Regression_pt: 0.0414 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0003 | Classification_binary: 0.4064 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3282 | Regression_eta: 0.0681 | Regression_pt: 0.3454 | Regression_sin_phi: 0.0814 | LR: 4.28e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4618s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0591 | max_tensorboard_logging: 0.1252 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3214.5 | uss_MB: 1986.3 | child_rss_MB: 22747.1 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:02:45,481] INFO : Running validation on rank0 +rank0 [2026-11-15 03:02:45,481] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:02:45,481] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:02:45,598] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 03:02:45,701] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:02:45,800] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:02:45,869] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:02:45,935] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:02:46,000] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:02:46,068] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:02:46,136] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:02:46,246] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:02:46,347] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:02:46,445] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:02:46,511] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:02:46,574] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:02:46,637] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:02:46,701] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:02:46,765] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:02:46,863] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:02:46,868] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:02:46,870] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:02:46,874] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:02:46,875] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:02:46,876] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:02:46,878] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:02:46,880] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:02:46,882] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:02:46,883] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:02:46,886] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:02:46,886] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:02:46,888] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:02:46,890] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:02:46,892] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:02:46,894] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:02:48,278] INFO : VALIDATION | Step=21000/500000 | Train Loss=1.2808 | Valid Loss=6.2011 | Train Loss Unweighted=0.4894 | Valid Loss Unweighted=1.5632 | Stale=10 | ETA=934.9m +rank0 [2026-11-15 03:02:48,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:02:48,350] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:02:48,356] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:02:48,402] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:02:48,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:02:49,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:02:49,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:02:49,601] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:02:49,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:02:50,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:02:50,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:02:50,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:02:50,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:02:51,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:02:51,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:02:51,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:02:51,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:02:51,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:02:52,222] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:02:52,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:02:52,494] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:02:52,499] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:02:53,027] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:02:53,369] INFO : Plotted number of elements +rank0 [2026-11-15 03:02:54,596] INFO : Plotted jets +rank0 [2026-11-15 03:02:55,923] INFO : Plotted jet ratio +rank0 [2026-11-15 03:03:33,825] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 03:03:33,825] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:04:33,425] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 03:04:33,425] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:04:42,523] INFO : Step 22000/500000 rank0 | Train Loss: 1.6328 | Train Loss Unweighted: 0.8123 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.7170 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0472 | Regression_eta: 0.0001 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0005 | Classification_binary: 0.7170 | Regression_cos_phi: 0.0443 | Regression_energy: 0.3749 | Regression_eta: 0.0381 | Regression_pt: 0.3998 | Regression_sin_phi: 0.0583 | LR: 4.50e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4770s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0032 | max_backward: 0.0399 | max_optimizer_step: 0.0594 | max_tensorboard_logging: 0.1233 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3237.8 | uss_MB: 2009.6 | child_rss_MB: 22746.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:04:42,525] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 03:04:42,590] INFO : Running validation on rank0 +rank0 [2026-11-15 03:04:42,590] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:04:42,590] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:04:42,706] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 03:04:42,808] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:04:42,907] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:04:42,974] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:04:43,039] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:04:43,103] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:04:43,169] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:04:43,234] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:04:43,343] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:04:43,445] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:04:43,546] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:04:43,613] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:04:43,679] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:04:43,744] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:04:43,810] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:04:43,877] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:04:43,977] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:04:43,983] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:04:43,984] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:04:43,986] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:04:43,989] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:04:43,990] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:04:43,991] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:04:43,992] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:04:43,994] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:04:43,997] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:04:43,999] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:04:44,001] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:04:44,002] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:04:44,002] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:04:44,006] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:04:44,006] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:04:45,385] INFO : VALIDATION | Step=22000/500000 | Train Loss=1.6328 | Valid Loss=6.4909 | Train Loss Unweighted=0.8123 | Valid Loss Unweighted=1.5919 | Stale=11 | ETA=933.0m +rank0 [2026-11-15 03:04:45,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:04:45,458] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:04:45,465] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:04:45,511] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:04:45,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:04:46,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:04:46,473] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:04:46,712] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:04:46,951] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:04:47,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:04:47,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:04:47,659] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:04:47,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:04:48,132] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:04:48,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:04:48,614] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:04:48,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:04:49,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:04:49,323] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:04:49,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:04:49,598] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:04:49,604] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:04:50,137] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:04:50,491] INFO : Plotted number of elements +rank0 [2026-11-15 03:04:51,706] INFO : Plotted jets +rank0 [2026-11-15 03:04:53,042] INFO : Plotted jet ratio +rank0 [2026-11-15 03:05:44,489] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 03:05:44,489] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:06:40,045] INFO : Step 23000/500000 rank0 | Train Loss: 1.3924 | Train Loss Unweighted: 0.6716 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.6249 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0226 | Regression_eta: 0.0003 | Regression_pt: 0.0232 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0347 | Classification_binary: 0.6249 | Regression_cos_phi: 0.1225 | Regression_energy: 0.1793 | Regression_eta: 0.1106 | Regression_pt: 0.1935 | Regression_sin_phi: 0.1270 | LR: 4.73e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4515s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0345 | max_loss: 0.0025 | max_backward: 0.0382 | max_optimizer_step: 0.0586 | max_tensorboard_logging: 0.1150 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3249.0 | uss_MB: 2020.9 | child_rss_MB: 22745.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:06:40,047] INFO : Running validation on rank0 +rank0 [2026-11-15 03:06:40,047] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:06:40,047] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:06:40,165] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 03:06:40,269] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:06:40,371] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:06:40,438] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:06:40,505] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:06:40,572] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:06:40,639] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:06:40,707] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:06:40,816] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:06:40,915] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:06:41,013] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:06:41,081] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:06:41,147] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:06:41,210] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:06:41,275] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:06:41,339] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:06:41,441] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:06:41,447] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:06:41,449] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:06:41,451] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:06:41,453] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:06:41,455] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:06:41,456] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:06:41,458] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:06:41,460] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:06:41,460] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:06:41,462] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:06:41,464] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:06:41,466] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:06:41,467] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:06:41,468] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:06:41,470] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:06:42,853] INFO : VALIDATION | Step=23000/500000 | Train Loss=1.3924 | Valid Loss=6.4271 | Train Loss Unweighted=0.6716 | Valid Loss Unweighted=1.6276 | Stale=12 | ETA=931.2m +rank0 [2026-11-15 03:06:42,854] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:06:42,922] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:06:42,928] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:06:42,974] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:06:43,448] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:06:43,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:06:43,944] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:06:44,186] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:06:44,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:06:44,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:06:44,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:06:45,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:06:45,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:06:45,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:06:45,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:06:46,121] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:06:46,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:06:46,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:06:46,830] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:06:47,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:06:47,125] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:06:47,130] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:06:47,660] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:06:48,012] INFO : Plotted number of elements +rank0 [2026-11-15 03:06:49,252] INFO : Plotted jets +rank0 [2026-11-15 03:06:50,600] INFO : Plotted jet ratio +rank0 [2026-11-15 03:06:55,352] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 03:06:55,352] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:07:55,171] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 03:07:55,171] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:08:37,399] INFO : Step 24000/500000 rank0 | Train Loss: 2.1361 | Train Loss Unweighted: 0.7894 | Train Loss Components: Classification: 0.0011 | Classification_binary: 0.7283 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0264 | Regression_eta: 0.0003 | Regression_pt: 0.0324 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0803 | Classification_binary: 0.7283 | Regression_cos_phi: 0.3155 | Regression_energy: 0.2101 | Regression_eta: 0.1175 | Regression_pt: 0.2707 | Regression_sin_phi: 0.4138 | LR: 4.97e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4557s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0026 | max_backward: 0.0385 | max_optimizer_step: 0.0599 | max_tensorboard_logging: 0.1147 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3283.7 | uss_MB: 2055.5 | child_rss_MB: 22745.4 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:08:37,401] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 03:08:37,466] INFO : Running validation on rank0 +rank0 [2026-11-15 03:08:37,466] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:08:37,466] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:08:37,582] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 03:08:37,684] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:08:37,782] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:08:37,849] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:08:37,915] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:08:37,980] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:08:38,048] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:08:38,114] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:08:38,226] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:08:38,329] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:08:38,430] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:08:38,496] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:08:38,560] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:08:38,624] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:08:38,688] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:08:38,751] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:08:38,850] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:08:38,854] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:08:38,856] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:08:38,860] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:08:38,861] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:08:38,863] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:08:38,864] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:08:38,866] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:08:38,868] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:08:38,870] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:08:38,870] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:08:38,872] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:08:38,874] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:08:38,876] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:08:38,877] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:08:38,880] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:08:40,254] INFO : VALIDATION | Step=24000/500000 | Train Loss=2.1361 | Valid Loss=6.7511 | Train Loss Unweighted=0.7894 | Valid Loss Unweighted=1.7583 | Stale=13 | ETA=929.3m +rank0 [2026-11-15 03:08:40,254] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:08:40,324] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:08:40,331] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:08:40,378] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:08:40,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:08:41,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:08:41,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:08:41,566] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:08:41,808] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:08:42,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:08:42,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:08:42,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:08:42,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:08:42,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:08:43,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:08:43,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:08:43,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:08:43,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:08:44,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:08:44,408] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:08:44,464] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:08:44,469] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:08:45,006] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:08:45,357] INFO : Plotted number of elements +rank0 [2026-11-15 03:08:46,584] INFO : Plotted jets +rank0 [2026-11-15 03:08:47,928] INFO : Plotted jet ratio +rank0 [2026-11-15 03:09:05,941] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 03:09:05,941] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:10:05,762] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 03:10:05,763] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:10:35,162] INFO : Step 25000/500000 rank0 | Train Loss: 0.6121 | Train Loss Unweighted: 0.1180 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0914 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0131 | Regression_eta: 0.0001 | Regression_pt: 0.0131 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0004 | Classification_binary: 0.0914 | Regression_cos_phi: 0.1352 | Regression_energy: 0.1037 | Regression_eta: 0.0520 | Regression_pt: 0.1091 | Regression_sin_phi: 0.1204 | LR: 5.22e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5664s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0024 | max_backward: 0.0380 | max_optimizer_step: 0.0586 | max_tensorboard_logging: 0.2311 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3310.6 | uss_MB: 2082.5 | child_rss_MB: 22742.9 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:10:35,164] INFO : Running validation on rank0 +rank0 [2026-11-15 03:10:35,164] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:10:35,164] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:10:35,292] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 03:10:35,397] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:10:35,503] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:10:35,571] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:10:35,638] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:10:35,703] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:10:35,769] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:10:35,835] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:10:35,947] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:10:36,049] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:10:36,150] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:10:36,216] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:10:36,282] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:10:36,346] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:10:36,411] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:10:36,475] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:10:36,580] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:10:36,586] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:10:36,587] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:10:36,589] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:10:36,592] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:10:36,594] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:10:36,595] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:10:36,597] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:10:36,598] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:10:36,601] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:10:36,602] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:10:36,604] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:10:36,605] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:10:36,607] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:10:36,608] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:10:36,611] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:10:38,012] INFO : VALIDATION | Step=25000/500000 | Train Loss=0.6121 | Valid Loss=6.7883 | Train Loss Unweighted=0.1180 | Valid Loss Unweighted=1.8468 | Stale=14 | ETA=927.5m +rank0 [2026-11-15 03:10:38,012] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:10:38,130] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:10:38,137] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:10:38,183] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:10:38,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:10:38,900] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:10:39,137] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:10:39,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:10:39,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:10:39,865] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:10:40,110] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:10:40,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:10:40,586] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:10:40,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:10:41,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:10:41,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:10:41,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:10:41,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:10:42,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:10:42,246] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:10:42,297] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:10:42,304] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:10:42,836] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:10:43,192] INFO : Plotted number of elements +rank0 [2026-11-15 03:10:44,440] INFO : Plotted jets +rank0 [2026-11-15 03:10:45,786] INFO : Plotted jet ratio +rank0 [2026-11-15 03:11:17,030] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 03:11:17,030] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:12:17,013] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 03:12:17,013] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:12:33,164] INFO : Step 26000/500000 rank0 | Train Loss: 0.5959 | Train Loss Unweighted: 0.0692 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0235 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0232 | Regression_eta: 0.0001 | Regression_pt: 0.0221 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0007 | Classification_binary: 0.0235 | Regression_cos_phi: 0.0732 | Regression_energy: 0.1847 | Regression_eta: 0.0532 | Regression_pt: 0.1847 | Regression_sin_phi: 0.0760 | LR: 5.47e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4839s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0730 | max_tensorboard_logging: 0.1280 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3319.9 | uss_MB: 2091.8 | child_rss_MB: 22742.9 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:12:33,166] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 03:12:33,234] INFO : Running validation on rank0 +rank0 [2026-11-15 03:12:33,234] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:12:33,234] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:12:33,354] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 03:12:33,458] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:12:33,558] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:12:33,624] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:12:33,689] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:12:33,753] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:12:33,820] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:12:33,886] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:12:33,997] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:12:34,100] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:12:34,200] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:12:34,268] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:12:34,333] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:12:34,399] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:12:34,466] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:12:34,531] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:12:34,637] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:12:34,642] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:12:34,645] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:12:34,648] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:12:34,650] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:12:34,651] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:12:34,652] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:12:34,654] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:12:34,656] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:12:34,658] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:12:34,660] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:12:34,661] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:12:34,662] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:12:34,663] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:12:34,666] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:12:34,668] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:12:36,064] INFO : VALIDATION | Step=26000/500000 | Train Loss=0.5959 | Valid Loss=6.7923 | Train Loss Unweighted=0.0692 | Valid Loss Unweighted=1.8443 | Stale=15 | ETA=925.9m +rank0 [2026-11-15 03:12:36,065] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:12:36,136] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:12:36,144] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:12:36,192] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:12:36,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:12:36,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:12:37,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:12:37,382] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:12:37,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:12:37,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:12:38,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:12:38,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:12:38,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:12:38,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:12:39,056] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:12:39,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:12:39,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:12:39,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:12:40,023] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:12:40,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:12:40,299] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:12:40,304] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:12:40,837] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:12:41,194] INFO : Plotted number of elements +rank0 [2026-11-15 03:12:42,421] INFO : Plotted jets +rank0 [2026-11-15 03:12:43,762] INFO : Plotted jet ratio +rank0 [2026-11-15 03:13:28,189] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 03:13:28,189] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:14:28,082] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 03:14:28,082] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:14:31,025] INFO : Step 27000/500000 rank0 | Train Loss: 1.1721 | Train Loss Unweighted: 0.2035 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0242 | Regression_eta: 0.0006 | Regression_pt: 0.0242 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0061 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0482 | Regression_energy: 0.1923 | Regression_eta: 0.2290 | Regression_pt: 0.2018 | Regression_sin_phi: 0.3408 | LR: 5.74e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4751s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0025 | max_backward: 0.0386 | max_optimizer_step: 0.0609 | max_tensorboard_logging: 0.1256 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3352.8 | uss_MB: 2124.6 | child_rss_MB: 22746.7 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:14:31,027] INFO : Running validation on rank0 +rank0 [2026-11-15 03:14:31,027] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:14:31,027] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:14:31,150] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 03:14:31,253] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:14:31,354] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:14:31,422] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:14:31,487] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:14:31,552] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:14:31,618] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:14:31,684] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:14:31,795] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:14:31,895] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:14:31,994] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:14:32,063] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:14:32,129] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:14:32,196] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:14:32,262] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:14:32,328] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:14:32,436] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:14:32,443] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:14:32,445] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:14:32,447] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:14:32,448] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:14:32,451] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:14:32,451] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:14:32,453] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:14:32,455] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:14:32,457] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:14:32,459] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:14:32,461] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:14:32,462] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:14:32,463] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:14:32,464] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:14:32,466] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:14:33,877] INFO : VALIDATION | Step=27000/500000 | Train Loss=1.1721 | Valid Loss=7.2023 | Train Loss Unweighted=0.2035 | Valid Loss Unweighted=1.8696 | Stale=16 | ETA=924.1m +rank0 [2026-11-15 03:14:33,877] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:14:33,949] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:14:33,955] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:14:34,004] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:14:34,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:14:34,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:14:34,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:14:35,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:14:35,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:14:35,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:14:35,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:14:36,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:14:36,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:14:36,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:14:36,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:14:37,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:14:37,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:14:37,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:14:37,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:14:38,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:14:38,095] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:14:38,101] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:14:38,629] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:14:38,982] INFO : Plotted number of elements +rank0 [2026-11-15 03:14:40,220] INFO : Plotted jets +rank0 [2026-11-15 03:14:41,570] INFO : Plotted jet ratio +rank0 [2026-11-15 03:15:39,218] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 03:15:39,218] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:16:28,758] INFO : Step 28000/500000 rank0 | Train Loss: 1.1354 | Train Loss Unweighted: 0.1801 | Train Loss Components: Classification: 0.0022 | Classification_binary: 0.1342 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0215 | Regression_eta: 0.0003 | Regression_pt: 0.0215 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.1630 | Classification_binary: 0.1342 | Regression_cos_phi: 0.0560 | Regression_energy: 0.1709 | Regression_eta: 0.0945 | Regression_pt: 0.1799 | Regression_sin_phi: 0.3369 | LR: 6.01e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4798s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0025 | max_backward: 0.0382 | max_optimizer_step: 0.0602 | max_tensorboard_logging: 0.1213 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3379.5 | uss_MB: 2151.3 | child_rss_MB: 22745.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:16:28,760] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 03:16:28,825] INFO : Running validation on rank0 +rank0 [2026-11-15 03:16:28,825] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:16:28,825] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:16:28,943] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 03:16:29,050] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:16:29,151] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:16:29,219] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:16:29,284] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:16:29,348] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:16:29,414] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:16:29,480] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:16:29,590] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:16:29,691] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:16:29,791] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:16:29,859] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:16:29,928] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:16:29,993] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:16:30,062] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:16:30,129] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:16:30,235] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:16:30,241] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:16:30,242] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:16:30,246] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:16:30,247] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:16:30,248] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:16:30,250] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:16:30,252] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:16:30,253] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:16:30,255] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:16:30,258] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:16:30,260] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:16:30,261] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:16:30,262] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:16:30,265] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:16:30,267] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:16:31,666] INFO : VALIDATION | Step=28000/500000 | Train Loss=1.1354 | Valid Loss=7.7273 | Train Loss Unweighted=0.1801 | Valid Loss Unweighted=2.0677 | Stale=17 | ETA=922.3m +rank0 [2026-11-15 03:16:31,666] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:16:31,737] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:16:31,744] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:16:31,791] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:16:32,254] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:16:32,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:16:32,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:16:32,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:16:33,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:16:33,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:16:33,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:16:33,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:16:34,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:16:34,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:16:34,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:16:34,895] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:16:35,139] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:16:35,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:16:35,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:16:35,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:16:35,891] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:16:35,896] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:16:36,426] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:16:36,779] INFO : Plotted number of elements +rank0 [2026-11-15 03:16:38,009] INFO : Plotted jets +rank0 [2026-11-15 03:16:39,357] INFO : Plotted jet ratio +rank0 [2026-11-15 03:16:50,336] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 03:16:50,336] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:17:50,216] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 03:17:50,217] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:18:26,690] INFO : Step 29000/500000 rank0 | Train Loss: 0.8729 | Train Loss Unweighted: 0.3614 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.3229 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0195 | Regression_eta: 0.0001 | Regression_pt: 0.0186 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0018 | Classification_binary: 0.3229 | Regression_cos_phi: 0.1202 | Regression_energy: 0.1548 | Regression_eta: 0.0469 | Regression_pt: 0.1553 | Regression_sin_phi: 0.0710 | LR: 6.29e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4845s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0357 | max_loss: 0.0026 | max_backward: 0.0437 | max_optimizer_step: 0.0631 | max_tensorboard_logging: 0.1240 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3405.8 | uss_MB: 2177.6 | child_rss_MB: 22745.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:18:26,692] INFO : Running validation on rank0 +rank0 [2026-11-15 03:18:26,692] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:18:26,692] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:18:26,809] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 03:18:26,912] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:18:27,013] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:18:27,083] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:18:27,149] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:18:27,214] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:18:27,279] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:18:27,345] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:18:27,456] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:18:27,558] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:18:27,658] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:18:27,726] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:18:27,792] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:18:27,857] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:18:27,924] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:18:27,991] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:18:28,098] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:18:28,103] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:18:28,107] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:18:28,109] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:18:28,110] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:18:28,113] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:18:28,114] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:18:28,116] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:18:28,118] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:18:28,120] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:18:28,122] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:18:28,124] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:18:28,125] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:18:28,126] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:18:28,129] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:18:28,130] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:18:29,530] INFO : VALIDATION | Step=29000/500000 | Train Loss=0.8729 | Valid Loss=7.8242 | Train Loss Unweighted=0.3614 | Valid Loss Unweighted=2.2341 | Stale=18 | ETA=920.5m +rank0 [2026-11-15 03:18:29,530] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:18:29,602] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:18:29,609] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:18:29,656] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:18:30,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:18:30,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:18:30,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:18:30,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:18:31,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:18:31,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:18:31,565] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:18:31,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:18:32,031] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:18:32,267] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:18:32,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:18:32,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:18:32,998] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:18:33,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:18:33,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:18:33,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:18:33,757] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:18:33,762] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:18:34,292] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:18:34,646] INFO : Plotted number of elements +rank0 [2026-11-15 03:18:35,877] INFO : Plotted jets +rank0 [2026-11-15 03:18:37,228] INFO : Plotted jet ratio +rank0 [2026-11-15 03:19:01,500] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 03:19:01,500] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:20:01,081] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 03:20:01,081] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:20:24,328] INFO : Step 30000/500000 rank0 | Train Loss: 0.8029 | Train Loss Unweighted: 0.1809 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.1504 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0150 | Regression_eta: 0.0002 | Regression_pt: 0.0148 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0115 | Classification_binary: 0.1504 | Regression_cos_phi: 0.1137 | Regression_energy: 0.1188 | Regression_eta: 0.0676 | Regression_pt: 0.1239 | Regression_sin_phi: 0.2170 | LR: 6.58e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4725s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0396 | max_optimizer_step: 0.0625 | max_tensorboard_logging: 0.1255 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3429.4 | uss_MB: 2201.3 | child_rss_MB: 22746.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:20:24,329] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 03:20:24,394] INFO : Running validation on rank0 +rank0 [2026-11-15 03:20:24,394] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:20:24,394] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:20:24,514] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 03:20:24,619] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:20:24,721] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:20:24,788] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:20:24,852] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:20:24,915] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:20:24,980] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:20:25,046] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:20:25,156] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:20:25,256] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:20:25,354] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:20:25,419] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:20:25,483] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:20:25,547] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:20:25,611] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:20:25,675] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:20:25,780] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:20:25,787] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:20:25,789] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:20:25,791] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:20:25,792] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:20:25,795] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:20:25,796] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:20:25,796] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:20:25,798] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:20:25,801] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:20:25,804] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:20:25,806] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:20:25,806] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:20:25,808] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:20:25,809] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:20:25,813] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:20:27,200] INFO : VALIDATION | Step=30000/500000 | Train Loss=0.8029 | Valid Loss=7.7593 | Train Loss Unweighted=0.1809 | Valid Loss Unweighted=2.1595 | Stale=19 | ETA=918.7m +rank0 [2026-11-15 03:20:27,201] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:20:27,275] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:20:27,283] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:27,330] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:20:27,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:20:28,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:20:28,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:20:28,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:20:28,756] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:20:28,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:20:29,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:20:29,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:20:29,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:20:29,953] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:20:30,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:20:30,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:20:30,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:20:30,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:20:31,176] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:20:31,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:20:31,455] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:20:31,461] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:31,986] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:32,340] INFO : Plotted number of elements +rank0 [2026-11-15 03:20:33,592] INFO : Plotted jets +rank0 [2026-11-15 03:20:34,940] INFO : Plotted jet ratio +rank0 [2026-11-15 03:21:12,447] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 03:21:12,447] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:22:12,413] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 03:22:12,413] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:22:22,388] INFO : Step 31000/500000 rank0 | Train Loss: 0.8978 | Train Loss Unweighted: 0.1203 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0605 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0288 | Regression_eta: 0.0002 | Regression_pt: 0.0305 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.0605 | Regression_cos_phi: 0.1069 | Regression_energy: 0.2286 | Regression_eta: 0.0608 | Regression_pt: 0.2546 | Regression_sin_phi: 0.1863 | LR: 6.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4884s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0405 | max_optimizer_step: 0.0622 | max_tensorboard_logging: 0.1329 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3380.7 | uss_MB: 2152.5 | child_rss_MB: 22746.2 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:22:22,390] INFO : Running validation on rank0 +rank0 [2026-11-15 03:22:22,390] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:22:22,390] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:22:22,507] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 03:22:22,614] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:22:22,714] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:22:22,782] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:22:22,848] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:22:22,912] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:22:22,977] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:22:23,044] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:22:23,157] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:22:23,258] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:22:23,358] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:22:23,426] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:22:23,492] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:22:23,558] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:22:23,625] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:22:23,692] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:22:23,800] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:22:23,807] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:22:23,808] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:22:23,811] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:22:23,812] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:22:23,815] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:22:23,816] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:22:23,818] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:22:23,820] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:22:23,822] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:22:23,822] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:22:23,826] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:22:23,826] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:22:23,828] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:22:23,830] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:22:23,832] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:22:25,236] INFO : VALIDATION | Step=31000/500000 | Train Loss=0.8978 | Valid Loss=8.0055 | Train Loss Unweighted=0.1203 | Valid Loss Unweighted=2.2648 | Stale=20 | ETA=916.9m +rank0 [2026-11-15 03:22:25,236] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:22:25,310] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:22:25,316] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:25,364] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:22:25,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:22:26,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:22:26,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:22:26,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:22:26,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:22:27,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:22:27,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:22:27,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:22:27,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:22:27,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:22:28,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:22:28,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:22:28,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:22:28,946] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:22:29,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:22:29,405] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:22:29,462] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:22:29,468] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:29,998] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:30,352] INFO : Plotted number of elements +rank0 [2026-11-15 03:22:31,591] INFO : Plotted jets +rank0 [2026-11-15 03:22:32,945] INFO : Plotted jet ratio +rank0 [2026-11-15 03:23:23,564] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-15 03:23:23,564] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:24:19,848] INFO : Step 32000/500000 rank0 | Train Loss: 1.0240 | Train Loss Unweighted: 0.1591 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.1016 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0286 | Regression_eta: 0.0002 | Regression_pt: 0.0283 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0058 | Classification_binary: 0.1016 | Regression_cos_phi: 0.1554 | Regression_energy: 0.2272 | Regression_eta: 0.0589 | Regression_pt: 0.2365 | Regression_sin_phi: 0.2387 | LR: 7.19e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4729s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0024 | max_backward: 0.0382 | max_optimizer_step: 0.0611 | max_tensorboard_logging: 0.1265 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3485.3 | uss_MB: 2257.1 | child_rss_MB: 22745.0 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:24:19,850] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-32000.pth +rank0 [2026-11-15 03:24:19,916] INFO : Running validation on rank0 +rank0 [2026-11-15 03:24:19,916] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:24:19,916] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:24:20,045] INFO : saved batch0_epoch32000.parquet +rank0 [2026-11-15 03:24:20,152] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:24:20,255] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:24:20,323] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:24:20,387] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:24:20,451] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:24:20,516] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:24:20,581] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:24:20,692] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:24:20,793] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:24:20,891] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:24:20,958] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:24:21,023] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:24:21,088] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:24:21,153] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:24:21,217] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:24:21,326] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:24:21,331] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:24:21,333] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:24:21,337] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:24:21,339] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:24:21,339] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:24:21,341] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:24:21,343] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:24:21,345] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:24:21,347] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:24:21,349] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:24:21,351] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:24:21,352] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:24:21,352] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:24:21,356] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:24:21,358] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:24:22,756] INFO : VALIDATION | Step=32000/500000 | Train Loss=1.0240 | Valid Loss=7.7241 | Train Loss Unweighted=0.1591 | Valid Loss Unweighted=2.3155 | Stale=21 | ETA=915.0m +rank0 [2026-11-15 03:24:22,757] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:24:22,830] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:24:22,838] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:22,886] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:24:23,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:24:23,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:24:23,865] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:24:24,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:24:24,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:24:24,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:24:24,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:24:25,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:24:25,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:24:25,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:24:25,785] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:24:26,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:24:26,277] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:24:26,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:24:26,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:24:26,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:24:27,032] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:24:27,038] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:27,565] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:27,920] INFO : Plotted number of elements +rank0 [2026-11-15 03:24:29,161] INFO : Plotted jets +rank0 [2026-11-15 03:24:30,549] INFO : Plotted jet ratio +rank0 [2026-11-15 03:24:34,508] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-15 03:24:34,508] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:25:34,402] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-15 03:25:34,402] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:26:17,781] INFO : Step 33000/500000 rank0 | Train Loss: 0.9192 | Train Loss Unweighted: 0.0681 | Train Loss Components: Classification: 0.0018 | Classification_binary: 0.0238 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0209 | Regression_eta: 0.0003 | Regression_pt: 0.0210 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.1336 | Classification_binary: 0.0238 | Regression_cos_phi: 0.1158 | Regression_energy: 0.1661 | Regression_eta: 0.1272 | Regression_pt: 0.1753 | Regression_sin_phi: 0.1774 | LR: 7.51e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4791s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0608 | max_tensorboard_logging: 0.1291 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3509.0 | uss_MB: 2280.9 | child_rss_MB: 22744.0 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:26:17,783] INFO : Running validation on rank0 +rank0 [2026-11-15 03:26:17,783] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:26:17,783] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:26:17,906] INFO : saved batch0_epoch33000.parquet +rank0 [2026-11-15 03:26:18,011] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:26:18,114] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:26:18,183] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:26:18,248] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:26:18,313] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:26:18,379] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:26:18,445] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:26:18,558] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:26:18,659] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:26:18,759] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:26:18,825] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:26:18,891] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:26:18,955] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:26:19,020] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:26:19,088] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:26:19,199] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:26:19,205] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:26:19,208] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:26:19,210] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:26:19,212] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:26:19,212] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:26:19,214] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:26:19,215] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:26:19,218] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:26:19,220] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:26:19,222] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:26:19,224] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:26:19,225] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:26:19,225] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:26:19,227] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:26:19,231] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:26:20,631] INFO : VALIDATION | Step=33000/500000 | Train Loss=0.9192 | Valid Loss=7.6376 | Train Loss Unweighted=0.0681 | Valid Loss Unweighted=2.4198 | Stale=22 | ETA=913.2m +rank0 [2026-11-15 03:26:20,631] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:26:20,705] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:26:20,711] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:20,760] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:26:21,246] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:26:21,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:26:21,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:26:21,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:26:22,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:26:22,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:26:22,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:26:22,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:26:23,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:26:23,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:26:23,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:26:23,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:26:24,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:26:24,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:26:24,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:26:24,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:26:24,894] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:26:24,900] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:25,432] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:25,782] INFO : Plotted number of elements +rank0 [2026-11-15 03:26:27,027] INFO : Plotted jets +rank0 [2026-11-15 03:26:28,387] INFO : Plotted jet ratio +rank0 [2026-11-15 03:26:45,505] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-15 03:26:45,505] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:27:45,172] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-15 03:27:45,173] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:28:15,238] INFO : Step 34000/500000 rank0 | Train Loss: 0.5505 | Train Loss Unweighted: 0.0645 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.0439 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0101 | Regression_eta: 0.0001 | Regression_pt: 0.0101 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0052 | Classification_binary: 0.0439 | Regression_cos_phi: 0.1097 | Regression_energy: 0.0802 | Regression_eta: 0.0442 | Regression_pt: 0.0840 | Regression_sin_phi: 0.1834 | LR: 7.83e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4810s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0024 | max_backward: 0.0383 | max_optimizer_step: 0.0607 | max_tensorboard_logging: 0.1324 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3453.7 | uss_MB: 2225.6 | child_rss_MB: 22740.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:28:15,240] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-34000.pth +rank0 [2026-11-15 03:28:15,306] INFO : Running validation on rank0 +rank0 [2026-11-15 03:28:15,306] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:28:15,306] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:28:15,432] INFO : saved batch0_epoch34000.parquet +rank0 [2026-11-15 03:28:15,540] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:28:15,640] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:28:15,709] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:28:15,775] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:28:15,839] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:28:15,905] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:28:15,971] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:28:16,086] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:28:16,188] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:28:16,287] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:28:16,354] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:28:16,419] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:28:16,485] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:28:16,551] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:28:16,617] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:28:16,728] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:28:16,735] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:28:16,737] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:28:16,739] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:28:16,741] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:28:16,743] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:28:16,744] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:28:16,746] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:28:16,746] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:28:16,748] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:28:16,750] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:28:16,752] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:28:16,754] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:28:16,756] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:28:16,758] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:28:16,759] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:28:18,167] INFO : VALIDATION | Step=34000/500000 | Train Loss=0.5505 | Valid Loss=8.3962 | Train Loss Unweighted=0.0645 | Valid Loss Unweighted=2.3623 | Stale=23 | ETA=911.3m +rank0 [2026-11-15 03:28:18,168] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:28:18,241] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:28:18,249] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:18,296] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:28:18,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:28:19,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:28:19,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:28:19,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:28:19,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:28:19,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:28:20,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:28:20,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:28:20,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:28:20,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:28:21,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:28:21,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:28:21,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:28:21,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:28:22,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:28:22,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:28:22,418] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:28:22,424] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:22,950] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:23,307] INFO : Plotted number of elements +rank0 [2026-11-15 03:28:24,536] INFO : Plotted jets +rank0 [2026-11-15 03:28:25,903] INFO : Plotted jet ratio +rank0 [2026-11-15 03:28:56,308] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-15 03:28:56,308] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:29:56,260] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-15 03:29:56,260] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:30:13,333] INFO : Step 35000/500000 rank0 | Train Loss: 1.6985 | Train Loss Unweighted: 0.4161 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.3627 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0252 | Regression_eta: 0.0013 | Regression_pt: 0.0263 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0165 | Classification_binary: 0.3627 | Regression_cos_phi: 0.1768 | Regression_energy: 0.1998 | Regression_eta: 0.4612 | Regression_pt: 0.2199 | Regression_sin_phi: 0.2616 | LR: 8.16e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4809s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0025 | max_backward: 0.0396 | max_optimizer_step: 0.0640 | max_tensorboard_logging: 0.1243 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3558.1 | uss_MB: 2329.9 | child_rss_MB: 22747.0 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:30:13,334] INFO : Running validation on rank0 +rank0 [2026-11-15 03:30:13,334] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:30:13,334] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:30:13,453] INFO : saved batch0_epoch35000.parquet +rank0 [2026-11-15 03:30:13,556] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:30:13,655] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:30:13,723] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:30:13,788] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:30:13,853] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:30:13,918] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:30:13,985] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:30:14,098] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:30:14,200] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:30:14,300] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:30:14,368] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:30:14,433] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:30:14,499] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:30:14,566] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:30:14,634] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:30:14,745] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:30:14,751] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:30:14,754] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:30:14,755] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:30:14,758] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:30:14,760] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:30:14,761] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:30:14,763] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:30:14,764] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:30:14,765] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:30:14,767] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:30:14,770] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:30:14,771] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:30:14,773] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:30:14,775] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:30:14,776] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:30:16,183] INFO : VALIDATION | Step=35000/500000 | Train Loss=1.6985 | Valid Loss=8.4200 | Train Loss Unweighted=0.4161 | Valid Loss Unweighted=2.5928 | Stale=24 | ETA=909.5m +rank0 [2026-11-15 03:30:16,183] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:30:16,255] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:30:16,261] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:16,309] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:30:16,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:30:17,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:30:17,254] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:30:17,496] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:30:17,737] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:30:17,979] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:30:18,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:30:18,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:30:18,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:30:18,924] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:30:19,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:30:19,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:30:19,654] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:30:19,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:30:20,133] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:30:20,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:30:20,409] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:30:20,415] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:20,948] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:21,307] INFO : Plotted number of elements +rank0 [2026-11-15 03:30:22,539] INFO : Plotted jets +rank0 [2026-11-15 03:30:23,905] INFO : Plotted jet ratio +rank0 [2026-11-15 03:31:07,580] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-15 03:31:07,580] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:32:07,428] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-15 03:32:07,428] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:32:11,302] INFO : Step 36000/500000 rank0 | Train Loss: 0.8054 | Train Loss Unweighted: 0.3554 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.3186 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0179 | Regression_eta: 0.0002 | Regression_pt: 0.0187 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0000 | Classification_binary: 0.3186 | Regression_cos_phi: 0.0416 | Regression_energy: 0.1418 | Regression_eta: 0.0733 | Regression_pt: 0.1559 | Regression_sin_phi: 0.0742 | LR: 8.50e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5724s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0344 | max_loss: 0.0024 | max_backward: 0.0380 | max_optimizer_step: 0.0598 | max_tensorboard_logging: 0.2263 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3581.0 | uss_MB: 2360.5 | child_rss_MB: 22746.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:32:11,305] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-36000.pth +rank0 [2026-11-15 03:32:11,376] INFO : Running validation on rank0 +rank0 [2026-11-15 03:32:11,376] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:32:11,376] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:32:11,497] INFO : saved batch0_epoch36000.parquet +rank0 [2026-11-15 03:32:11,600] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:32:11,699] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:32:11,766] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:32:11,831] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:32:11,894] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:32:11,959] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:32:12,025] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:32:12,139] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:32:12,239] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:32:12,341] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:32:12,407] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:32:12,472] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:32:12,538] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:32:12,605] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:32:12,672] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:32:12,786] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:32:12,793] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:32:12,795] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:32:12,797] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:32:12,799] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:32:12,801] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:32:12,802] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:32:12,804] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:32:12,805] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:32:12,808] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:32:12,809] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:32:12,810] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:32:12,812] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:32:12,814] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:32:12,816] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:32:12,820] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:32:14,219] INFO : VALIDATION | Step=36000/500000 | Train Loss=0.8054 | Valid Loss=8.4207 | Train Loss Unweighted=0.3554 | Valid Loss Unweighted=2.5628 | Stale=25 | ETA=907.6m +rank0 [2026-11-15 03:32:14,219] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:32:14,333] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:32:14,340] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:32:14,395] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:32:14,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:32:15,099] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:32:15,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:32:15,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:32:15,831] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:32:16,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:32:16,319] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:32:16,555] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:32:16,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:32:17,031] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:32:17,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:32:17,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:32:17,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:32:18,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:32:18,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:32:18,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:32:18,527] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:32:18,534] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:32:19,068] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:32:19,423] INFO : Plotted number of elements +rank0 [2026-11-15 03:32:20,658] INFO : Plotted jets +rank0 [2026-11-15 03:32:22,045] INFO : Plotted jet ratio +rank0 [2026-11-15 03:33:18,892] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-15 03:33:18,892] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:34:09,189] INFO : Step 37000/500000 rank0 | Train Loss: 1.0400 | Train Loss Unweighted: 0.2912 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.2189 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0357 | Regression_eta: 0.0002 | Regression_pt: 0.0362 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.2189 | Regression_cos_phi: 0.1056 | Regression_energy: 0.2834 | Regression_eta: 0.0662 | Regression_pt: 0.3026 | Regression_sin_phi: 0.0631 | LR: 8.85e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4882s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0359 | max_loss: 0.0027 | max_backward: 0.0387 | max_optimizer_step: 0.0651 | max_tensorboard_logging: 0.1195 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3602.1 | uss_MB: 2374.2 | child_rss_MB: 22745.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:34:09,190] INFO : Running validation on rank0 +rank0 [2026-11-15 03:34:09,191] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:34:09,191] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:34:09,310] INFO : saved batch0_epoch37000.parquet +rank0 [2026-11-15 03:34:09,415] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:34:09,517] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:34:09,584] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:34:09,649] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:34:09,714] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:34:09,781] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:34:09,849] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:34:09,963] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:34:10,066] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:34:10,196] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:34:10,264] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:34:10,330] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:34:10,398] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:34:10,463] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:34:10,527] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:34:10,637] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:34:10,643] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:34:10,644] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:34:10,646] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:34:10,648] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:34:10,650] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:34:10,652] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:34:10,654] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:34:10,656] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:34:10,658] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:34:10,660] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:34:10,662] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:34:10,662] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:34:10,664] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:34:10,666] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:34:10,668] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:34:12,063] INFO : VALIDATION | Step=37000/500000 | Train Loss=1.0400 | Valid Loss=8.3483 | Train Loss Unweighted=0.2912 | Valid Loss Unweighted=2.7325 | Stale=26 | ETA=905.8m +rank0 [2026-11-15 03:34:12,064] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:34:12,132] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:34:12,137] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:34:12,186] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:34:12,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:34:12,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:34:13,161] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:34:13,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:34:13,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:34:13,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:34:14,115] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:34:14,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:34:14,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:34:14,829] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:34:15,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:34:15,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:34:15,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:34:15,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:34:16,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:34:16,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:34:16,289] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:34:16,295] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:34:16,830] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:34:17,182] INFO : Plotted number of elements +rank0 [2026-11-15 03:34:18,402] INFO : Plotted jets +rank0 [2026-11-15 03:34:19,755] INFO : Plotted jet ratio +rank0 [2026-11-15 03:34:29,819] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-15 03:34:29,819] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:35:29,345] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-15 03:35:29,345] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:36:06,372] INFO : Step 38000/500000 rank0 | Train Loss: 0.9542 | Train Loss Unweighted: 0.0576 | Train Loss Components: Classification: 0.0008 | Classification_binary: 0.0257 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0152 | Regression_eta: 0.0002 | Regression_pt: 0.0151 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.0257 | Regression_cos_phi: 0.2645 | Regression_energy: 0.1211 | Regression_eta: 0.0791 | Regression_pt: 0.1258 | Regression_sin_phi: 0.2816 | LR: 9.21e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4670s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0026 | max_backward: 0.0394 | max_optimizer_step: 0.0617 | max_tensorboard_logging: 0.1153 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3630.8 | uss_MB: 2402.9 | child_rss_MB: 22745.1 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:36:06,374] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-38000.pth +rank0 [2026-11-15 03:36:06,442] INFO : Running validation on rank0 +rank0 [2026-11-15 03:36:06,442] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:36:06,442] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:36:06,561] INFO : saved batch0_epoch38000.parquet +rank0 [2026-11-15 03:36:06,664] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:36:06,767] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:36:06,833] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:36:06,897] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:36:06,960] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:36:07,026] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:36:07,093] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:36:07,204] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:36:07,304] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:36:07,402] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:36:07,468] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:36:07,532] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:36:07,596] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:36:07,660] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:36:07,724] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:36:07,832] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:36:07,839] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:36:07,839] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:36:07,843] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:36:07,844] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:36:07,847] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:36:07,847] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:36:07,848] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:36:07,851] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:36:07,852] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:36:07,855] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:36:07,857] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:36:07,858] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:36:07,858] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:36:07,860] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:36:07,864] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:36:09,251] INFO : VALIDATION | Step=38000/500000 | Train Loss=0.9542 | Valid Loss=9.3149 | Train Loss Unweighted=0.0576 | Valid Loss Unweighted=2.8578 | Stale=27 | ETA=903.8m +rank0 [2026-11-15 03:36:09,252] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:36:09,321] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:36:09,324] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:36:09,372] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:36:09,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:36:10,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:36:10,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:36:10,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:36:10,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:36:11,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:36:11,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:36:11,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:36:11,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:36:11,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:36:12,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:36:12,475] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:36:12,716] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:36:12,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:36:13,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:36:13,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:36:13,476] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:36:13,482] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:36:14,011] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:36:14,365] INFO : Plotted number of elements +rank0 [2026-11-15 03:36:15,596] INFO : Plotted jets +rank0 [2026-11-15 03:36:16,961] INFO : Plotted jet ratio +rank0 [2026-11-15 03:36:40,353] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-15 03:36:40,353] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:37:39,953] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-15 03:37:39,953] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:38:04,058] INFO : Step 39000/500000 rank0 | Train Loss: 0.6835 | Train Loss Unweighted: 0.1301 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0964 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0168 | Regression_eta: 0.0002 | Regression_pt: 0.0164 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0000 | Classification_binary: 0.0964 | Regression_cos_phi: 0.1282 | Regression_energy: 0.1336 | Regression_eta: 0.0554 | Regression_pt: 0.1373 | Regression_sin_phi: 0.1326 | LR: 9.57e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4995s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0026 | max_backward: 0.0395 | max_optimizer_step: 0.0628 | max_tensorboard_logging: 0.1231 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3654.1 | uss_MB: 2426.2 | child_rss_MB: 22746.9 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:38:04,060] INFO : Running validation on rank0 +rank0 [2026-11-15 03:38:04,060] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:38:04,060] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:38:04,177] INFO : saved batch0_epoch39000.parquet +rank0 [2026-11-15 03:38:04,283] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:38:04,383] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:38:04,452] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:38:04,517] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:38:04,580] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:38:04,645] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:38:04,710] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:38:04,822] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:38:04,925] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:38:05,025] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:38:05,094] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:38:05,160] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:38:05,225] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:38:05,292] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:38:05,359] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:38:05,472] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:38:05,479] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:38:05,479] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:38:05,483] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:38:05,484] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:38:05,487] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:38:05,488] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:38:05,490] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:38:05,492] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:38:05,494] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:38:05,496] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:38:05,498] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:38:05,499] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:38:05,501] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:38:05,501] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:38:05,503] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:38:06,893] INFO : VALIDATION | Step=39000/500000 | Train Loss=0.6835 | Valid Loss=8.2446 | Train Loss Unweighted=0.1301 | Valid Loss Unweighted=2.7478 | Stale=28 | ETA=901.9m +rank0 [2026-11-15 03:38:06,894] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:38:06,965] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:38:06,968] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:38:07,017] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:38:07,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:38:07,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:38:07,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:38:08,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:38:08,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:38:08,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:38:08,946] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:38:09,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:38:09,417] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:38:09,654] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:38:09,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:38:10,140] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:38:10,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:38:10,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:38:10,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:38:11,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:38:11,143] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:38:11,148] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:38:11,675] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:38:12,029] INFO : Plotted number of elements +rank0 [2026-11-15 03:38:13,273] INFO : Plotted jets +rank0 [2026-11-15 03:38:14,651] INFO : Plotted jet ratio +rank0 [2026-11-15 03:38:51,237] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-15 03:38:51,237] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:39:50,979] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-15 03:39:50,979] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:40:01,787] INFO : Step 40000/500000 rank0 | Train Loss: 1.0405 | Train Loss Unweighted: 0.0667 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0161 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0252 | Regression_eta: 0.0002 | Regression_pt: 0.0246 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0002 | Classification_binary: 0.0161 | Regression_cos_phi: 0.3844 | Regression_energy: 0.2000 | Regression_eta: 0.0760 | Regression_pt: 0.2056 | Regression_sin_phi: 0.1581 | LR: 9.94e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4705s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0617 | max_tensorboard_logging: 0.1205 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3686.5 | uss_MB: 2458.7 | child_rss_MB: 22745.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:40:01,789] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-40000.pth +rank0 [2026-11-15 03:40:01,856] INFO : Running validation on rank0 +rank0 [2026-11-15 03:40:01,856] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:40:01,856] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:40:01,974] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-15 03:40:02,078] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:40:02,182] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:40:02,250] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:40:02,317] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:40:02,383] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:40:02,451] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:40:02,521] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:40:02,636] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:40:02,737] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:40:02,837] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:40:02,903] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:40:02,969] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:40:03,034] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:40:03,101] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:40:03,165] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:40:03,273] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:40:03,278] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:40:03,282] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:40:03,283] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:40:03,284] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:40:03,286] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:40:03,288] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:40:03,290] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:40:03,292] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:40:03,294] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:40:03,296] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:40:03,298] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:40:03,299] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:40:03,301] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:40:03,302] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:40:03,305] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:40:04,697] INFO : VALIDATION | Step=40000/500000 | Train Loss=1.0405 | Valid Loss=8.6351 | Train Loss Unweighted=0.0667 | Valid Loss Unweighted=2.7612 | Stale=29 | ETA=900.0m +rank0 [2026-11-15 03:40:04,697] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:40:04,768] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:40:04,772] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:40:04,823] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:40:05,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:40:05,607] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:40:05,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:40:06,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:40:06,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:40:06,554] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:40:06,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:40:07,047] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:40:07,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:40:07,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:40:07,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:40:08,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:40:08,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:40:08,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:40:08,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:40:08,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:40:08,991] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:40:08,996] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:40:09,526] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:40:09,866] INFO : Plotted number of elements +rank0 [2026-11-15 03:40:11,105] INFO : Plotted jets +rank0 [2026-11-15 03:40:12,458] INFO : Plotted jet ratio +rank0 [2026-11-15 03:41:02,112] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-15 03:41:02,112] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:41:59,390] INFO : Step 41000/500000 rank0 | Train Loss: 1.0021 | Train Loss Unweighted: 0.1107 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.0625 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0238 | Regression_eta: 0.0003 | Regression_pt: 0.0234 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0122 | Classification_binary: 0.0625 | Regression_cos_phi: 0.2292 | Regression_energy: 0.1894 | Regression_eta: 0.1114 | Regression_pt: 0.1952 | Regression_sin_phi: 0.2022 | LR: 1.03e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.4810s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0354 | max_loss: 0.0026 | max_backward: 0.0390 | max_optimizer_step: 0.0622 | max_tensorboard_logging: 0.1244 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3695.8 | uss_MB: 2468.0 | child_rss_MB: 22745.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:41:59,392] INFO : Running validation on rank0 +rank0 [2026-11-15 03:41:59,392] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:41:59,392] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:41:59,513] INFO : saved batch0_epoch41000.parquet +rank0 [2026-11-15 03:41:59,618] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:41:59,717] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:41:59,785] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:41:59,850] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:41:59,915] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:41:59,980] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:42:00,047] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:42:00,161] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:42:00,261] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:42:00,360] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:42:00,426] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:42:00,492] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:42:00,555] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:42:00,621] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:42:00,686] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:42:00,797] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:42:00,802] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:42:00,804] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:42:00,807] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:42:00,809] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:42:00,811] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:42:00,812] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:42:00,814] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:42:00,814] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:42:00,816] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:42:00,818] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:42:00,820] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:42:00,822] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:42:00,824] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:42:00,826] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:42:00,828] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:42:02,240] INFO : VALIDATION | Step=41000/500000 | Train Loss=1.0021 | Valid Loss=8.3927 | Train Loss Unweighted=0.1107 | Valid Loss Unweighted=2.7661 | Stale=30 | ETA=898.1m +rank0 [2026-11-15 03:42:02,241] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:42:02,313] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:42:02,316] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:42:02,365] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:42:02,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:42:03,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:42:03,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:42:03,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:42:03,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:42:04,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:42:04,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:42:04,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:42:04,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:42:04,991] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:42:05,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:42:05,474] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:42:05,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:42:05,952] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:42:06,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:42:06,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:42:06,469] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:42:06,485] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:42:07,012] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:42:07,366] INFO : Plotted number of elements +rank0 [2026-11-15 03:42:08,613] INFO : Plotted jets +rank0 [2026-11-15 03:42:09,982] INFO : Plotted jet ratio +rank0 [2026-11-15 03:42:10,319] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 03:42:10,320] INFO : Training completed. Total time on device 0: 80.356min diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..7b78e45a4f2f9be73e5e399e02a8e773d0046cf8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/train.log.0 @@ -0,0 +1,3164 @@ +rank0 [2026-11-15 02:21:45,906] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 02:21:45,906] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 02:21:45,907] INFO : Instantiating model +rank0 [2026-11-15 02:21:45,933] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 02:21:45,933] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 02:21:45,935] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 02:21:45,935] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 02:21:45,935] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 02:21:45,935] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 02:21:45,935] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,938] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 02:21:45,938] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,940] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 02:21:45,940] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,943] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 02:21:45,943] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,946] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 02:21:45,946] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,948] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 02:21:45,948] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,951] INFO : Backbone initialization took 0.02s +rank0 [2026-11-15 02:21:45,951] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 02:21:45,951] INFO : Initializing output DNNs +rank0 [2026-11-15 02:21:45,965] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,968] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,975] INFO : Output DNNs initialization took 0.02s +rank0 [2026-11-15 02:21:45,976] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 02:21:45,976] INFO : jagged_attention enabled=True +rank0 [2026-11-15 02:21:45,976] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 02:21:45,976] INFO : task_queries enabled=True +rank0 [2026-11-15 02:21:45,976] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 02:21:45,976] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 02:21:45,976] INFO : classification_query parameters: 256 +rank0 [2026-11-15 02:21:45,976] INFO : regression_query parameters: 256 +rank0 [2026-11-15 02:21:45,976] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 02:21:45,976] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 02:21:45,976] INFO : output_mode=elementwise +rank0 [2026-11-15 02:21:45,976] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 02:21:45,976] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 02:21:45,976] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 02:21:45,976] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 02:21:45,976] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 02:21:45,976] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 02:21:45,976] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 02:21:45,977] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 02:21:45,978] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 02:21:45,978] INFO : MLPF __init__ done +rank0 [2026-11-15 02:21:45,978] INFO : Instantiated model +rank0 [2026-11-15 02:21:45,978] INFO : Moving model to device rank=0 +rank0 [2026-11-15 02:21:45,989] INFO : Moved model to device rank=0 +rank0 [2026-11-15 02:21:45,990] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 02:21:46,117] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 02:21:46,117] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 02:21:46,117] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 02:21:46,215] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 02:21:46,215] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 02:21:46,216] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 02:21:46,216] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 02:21:46,216] INFO : Total parameters: 3910930 +rank0 [2026-11-15 02:21:46,218] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 02:21:46,224] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398 +rank0 [2026-11-15 02:21:46,224] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398 +rank0 [2026-11-15 02:21:46,224] INFO : Getting dataloaders +rank0 [2026-11-15 02:21:46,224] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:21:48,105] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,160] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 02:21:48,328] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 02:21:48,410] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,451] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 02:21:48,528] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,573] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,606] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,656] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 02:21:48,695] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,695] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 02:21:48,696] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 02:21:48,696] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-15 02:21:48,696] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:21:48,703] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,710] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,717] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,724] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,731] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,738] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,746] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,752] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,763] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,770] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,770] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 02:21:48,770] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 02:21:48,799] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:21:48,950] INFO : Got dataloaders +rank0 [2026-11-15 02:21:48,951] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-15 02:21:48,951] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-15 02:21:48,951] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 02:21:48,958] INFO : Creating train_iterator +rank0 [2026-11-15 02:21:48,958] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:21:48,983] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:21:49,139] INFO : Created train_iterator +rank0 [2026-11-15 02:22:01,445] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.7744583451747895, 'Classification': 0.023724770015105603, 'Regression_pt': 0.21244246032088995, 'Regression_eta': 0.0048217891092645, 'Regression_sin_phi': 0.001596466990886256, 'Regression_cos_phi': 0.0022173133328033147, 'Regression_energy': 0.22335495825856924}, weights={'Classification_binary': 1.0, 'Classification': 74.79348754882812, 'Regression_pt': 8.352653503417969, 'Regression_eta': 368.00830078125, 'Regression_sin_phi': 1000.0, 'Regression_cos_phi': 800.2740478515625, 'Regression_energy': 7.94456672668457} +rank0 [2026-11-15 02:22:50,708] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 02:22:50,708] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:23:37,470] INFO : Step 1000/500000 rank0 | Train Loss: 5.6218 | Train Loss Unweighted: 0.9890 | Train Loss Components: Classification: 0.0060 | Classification_binary: 0.5879 | Regression_cos_phi: 0.0004 | Regression_energy: 0.1936 | Regression_eta: 0.0019 | Regression_pt: 0.1989 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.4469 | Classification_binary: 0.5879 | Regression_cos_phi: 0.3096 | Regression_energy: 1.5382 | Regression_eta: 0.6922 | Regression_pt: 1.6612 | Regression_sin_phi: 0.3858 | LR: 2.01e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4571s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0002 | max_forward: 0.0362 | max_loss: 0.0028 | max_backward: 0.0404 | max_optimizer_step: 0.0561 | max_tensorboard_logging: 0.1385 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2577.5 | uss_MB: 1341.7 | child_rss_MB: 22955.9 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:23:37,471] INFO : Running validation on rank0 +rank0 [2026-11-15 02:23:37,472] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:23:37,472] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:23:37,610] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 02:23:37,793] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:23:37,860] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:23:37,912] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:23:37,965] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:23:38,011] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:23:38,065] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:23:38,119] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:23:38,168] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:23:38,232] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:23:38,297] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:23:38,349] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:23:38,403] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:23:38,449] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:23:38,504] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:23:38,558] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:23:38,608] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:23:38,611] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:23:38,614] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:23:38,616] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:23:38,618] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:23:38,620] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:23:38,621] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:23:38,623] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:23:38,623] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:23:38,626] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:23:38,627] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:23:38,630] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:23:38,631] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:23:38,633] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:23:38,633] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:23:38,637] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:23:40,100] INFO : VALIDATION | Step=1000/500000 | Train Loss=5.6218 | Valid Loss=5.4613 | Train Loss Unweighted=0.9890 | Valid Loss Unweighted=1.2668 | Stale=0 | ETA=924.3m +rank0 [2026-11-15 02:23:40,101] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:23:40,175] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:23:40,180] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:23:40,227] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:23:40,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:23:41,221] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:23:41,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:23:41,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:23:41,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:23:42,188] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:23:42,432] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:23:42,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:23:42,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:23:43,149] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:23:43,399] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:23:43,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:23:43,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:23:44,113] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:23:44,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:23:44,562] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:23:44,609] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:23:44,615] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:23:45,151] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_1000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:23:45,988] INFO : Plotted number of elements +rank0 [2026-11-15 02:23:47,270] INFO : Plotted jets +rank0 [2026-11-15 02:23:48,560] INFO : Plotted jet ratio +rank0 [2026-11-15 02:24:01,953] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 02:24:01,953] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:25:01,305] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 02:25:01,305] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:25:34,810] INFO : Step 2000/500000 rank0 | Train Loss: 3.1491 | Train Loss Unweighted: 1.3639 | Train Loss Components: Classification: 0.0031 | Classification_binary: 1.2445 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0575 | Regression_eta: 0.0004 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.2287 | Classification_binary: 1.2445 | Regression_cos_phi: 0.4398 | Regression_energy: 0.4572 | Regression_eta: 0.1290 | Regression_pt: 0.4828 | Regression_sin_phi: 0.1671 | LR: 2.02e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4497s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0380 | max_optimizer_step: 0.0570 | max_tensorboard_logging: 0.1291 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2698.8 | uss_MB: 1470.4 | child_rss_MB: 22746.4 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:25:34,812] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 02:25:34,879] INFO : Running validation on rank0 +rank0 [2026-11-15 02:25:34,879] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:25:34,879] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:25:34,996] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 02:25:35,103] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:25:35,202] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:25:35,269] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:25:35,333] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:25:35,396] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:25:35,461] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:25:35,526] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:25:35,633] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:25:35,732] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:25:35,830] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:25:35,896] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:25:35,962] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:25:36,028] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:25:36,095] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:25:36,160] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:25:36,269] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:25:36,276] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:25:36,278] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:25:36,280] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:25:36,281] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:25:36,284] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:25:36,285] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:25:36,285] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:25:36,288] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:25:36,289] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:25:36,292] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:25:36,294] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:25:36,295] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:25:36,295] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:25:36,299] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:25:36,301] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:25:37,753] INFO : VALIDATION | Step=2000/500000 | Train Loss=3.1491 | Valid Loss=5.3821 | Train Loss Unweighted=1.3639 | Valid Loss Unweighted=1.2212 | Stale=0 | ETA=949.5m +rank0 [2026-11-15 02:25:37,753] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:25:37,828] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:25:37,835] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:25:37,878] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:25:38,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:25:38,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:25:38,827] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:25:39,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:25:39,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:25:39,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:25:39,783] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:25:40,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:25:40,253] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:25:40,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:25:40,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:25:40,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:25:41,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:25:41,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:25:41,698] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:25:41,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:25:41,960] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:25:41,965] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:25:42,491] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_2000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:25:42,838] INFO : Plotted number of elements +rank0 [2026-11-15 02:25:44,458] INFO : Plotted jets +rank0 [2026-11-15 02:25:45,381] INFO : Plotted jet ratio +rank0 [2026-11-15 02:26:11,917] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 02:26:11,917] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:27:11,174] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 02:27:11,174] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:27:31,631] INFO : Step 3000/500000 rank0 | Train Loss: 3.4370 | Train Loss Unweighted: 1.3986 | Train Loss Components: Classification: 0.0044 | Classification_binary: 1.2261 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0833 | Regression_eta: 0.0004 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.3269 | Classification_binary: 1.2261 | Regression_cos_phi: 0.1289 | Regression_energy: 0.6615 | Regression_eta: 0.1409 | Regression_pt: 0.7026 | Regression_sin_phi: 0.2501 | LR: 2.05e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4460s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0579 | max_tensorboard_logging: 0.1169 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2754.1 | uss_MB: 1525.9 | child_rss_MB: 22746.2 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:27:31,633] INFO : Running validation on rank0 +rank0 [2026-11-15 02:27:31,633] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:27:31,633] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:27:31,750] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 02:27:31,855] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:27:31,955] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:27:32,022] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:27:32,089] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:27:32,155] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:27:32,221] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:27:32,289] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:27:32,397] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:27:32,499] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:27:32,601] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:27:32,668] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:27:32,733] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:27:32,796] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:27:32,861] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:27:32,925] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:27:33,038] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:27:33,044] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:27:33,046] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:27:33,049] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:27:33,049] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:27:33,053] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:27:33,053] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:27:33,055] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:27:33,057] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:27:33,059] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:27:33,061] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:27:33,063] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:27:33,064] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:27:33,064] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:27:33,068] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:27:33,070] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:27:34,529] INFO : VALIDATION | Step=3000/500000 | Train Loss=3.4370 | Valid Loss=5.0607 | Train Loss Unweighted=1.3986 | Valid Loss Unweighted=1.2039 | Stale=0 | ETA=954.2m +rank0 [2026-11-15 02:27:34,530] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:27:34,600] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:27:34,607] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:27:34,651] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:27:35,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:27:35,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:27:35,578] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:27:35,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:27:36,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:27:36,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:27:36,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:27:36,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:27:37,029] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:27:37,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:27:37,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:27:37,763] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:27:38,003] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:27:38,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:27:38,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:27:38,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:27:38,735] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:27:38,741] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:27:39,278] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_3000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:27:39,627] INFO : Plotted number of elements +rank0 [2026-11-15 02:27:40,855] INFO : Plotted jets +rank0 [2026-11-15 02:27:42,186] INFO : Plotted jet ratio +rank0 [2026-11-15 02:28:22,050] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 02:28:22,050] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:29:21,702] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 02:29:21,702] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:29:29,123] INFO : Step 4000/500000 rank0 | Train Loss: 4.8232 | Train Loss Unweighted: 1.0335 | Train Loss Components: Classification: 0.0021 | Classification_binary: 0.8629 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0807 | Regression_eta: 0.0045 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.1572 | Classification_binary: 0.8629 | Regression_cos_phi: 0.4461 | Regression_energy: 0.6408 | Regression_eta: 1.6570 | Regression_pt: 0.6889 | Regression_sin_phi: 0.3704 | LR: 2.08e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4716s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0404 | max_loss: 0.0028 | max_backward: 0.0400 | max_optimizer_step: 0.0598 | max_tensorboard_logging: 0.1271 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2787.9 | uss_MB: 1559.7 | child_rss_MB: 22744.9 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:29:29,125] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 02:29:29,191] INFO : Running validation on rank0 +rank0 [2026-11-15 02:29:29,191] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:29:29,191] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:29:29,310] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 02:29:29,414] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:29:29,514] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:29:29,581] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:29:29,646] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:29:29,709] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:29:29,775] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:29:29,841] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:29:29,947] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:29:30,049] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:29:30,153] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:29:30,220] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:29:30,285] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:29:30,350] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:29:30,417] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:29:30,483] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:29:30,579] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:29:30,584] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:29:30,585] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:29:30,589] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:29:30,589] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:29:30,591] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:29:30,593] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:29:30,595] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:29:30,597] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:29:30,599] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:29:30,601] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:29:30,603] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:29:30,603] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:29:30,605] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:29:30,607] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:29:30,609] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:29:32,073] INFO : VALIDATION | Step=4000/500000 | Train Loss=4.8232 | Valid Loss=4.9634 | Train Loss Unweighted=1.0335 | Valid Loss Unweighted=1.1719 | Stale=0 | ETA=957.1m +rank0 [2026-11-15 02:29:32,074] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:29:32,146] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:29:32,152] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:29:32,196] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:29:32,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:29:32,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:29:33,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:29:33,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:29:33,600] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:29:33,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:29:34,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:29:34,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:29:34,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:29:34,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:29:35,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:29:35,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:29:35,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:29:35,765] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:29:35,999] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:29:36,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:29:36,268] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:29:36,274] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:29:36,808] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_4000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:29:37,167] INFO : Plotted number of elements +rank0 [2026-11-15 02:29:38,401] INFO : Plotted jets +rank0 [2026-11-15 02:29:39,739] INFO : Plotted jet ratio +rank0 [2026-11-15 02:30:32,992] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 02:30:32,992] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:31:27,154] INFO : Step 5000/500000 rank0 | Train Loss: 2.7700 | Train Loss Unweighted: 0.9656 | Train Loss Components: Classification: 0.0065 | Classification_binary: 0.8477 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0559 | Regression_eta: 0.0004 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.4895 | Classification_binary: 0.8477 | Regression_cos_phi: 0.1431 | Regression_energy: 0.4445 | Regression_eta: 0.1579 | Regression_pt: 0.4556 | Regression_sin_phi: 0.2317 | LR: 2.13e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4524s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0582 | max_tensorboard_logging: 0.1227 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2811.2 | uss_MB: 1583.0 | child_rss_MB: 22744.5 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:31:27,156] INFO : Running validation on rank0 +rank0 [2026-11-15 02:31:27,156] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:31:27,156] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:31:27,282] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 02:31:27,388] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:31:27,490] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:31:27,559] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:31:27,626] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:31:27,692] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:31:27,758] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:31:27,823] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:31:27,930] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:31:28,031] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:31:28,132] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:31:28,199] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:31:28,262] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:31:28,325] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:31:28,390] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:31:28,454] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:31:28,565] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:31:28,572] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:31:28,574] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:31:28,576] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:31:28,576] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:31:28,580] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:31:28,580] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:31:28,582] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:31:28,584] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:31:28,585] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:31:28,588] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:31:28,590] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:31:28,590] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:31:28,592] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:31:28,594] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:31:28,595] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:31:30,061] INFO : VALIDATION | Step=5000/500000 | Train Loss=2.7700 | Valid Loss=4.9318 | Train Loss Unweighted=0.9656 | Valid Loss Unweighted=1.1745 | Stale=0 | ETA=958.8m +rank0 [2026-11-15 02:31:30,062] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:31:30,134] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:31:30,140] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:31:30,180] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:31:30,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:31:30,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:31:31,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:31:31,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:31:31,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:31:31,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:31:32,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:31:32,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:31:32,558] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:31:32,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:31:33,027] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:31:33,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:31:33,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:31:33,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:31:33,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:31:34,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:31:34,255] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:31:34,260] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:31:34,787] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_5000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:31:35,128] INFO : Plotted number of elements +rank0 [2026-11-15 02:31:36,359] INFO : Plotted jets +rank0 [2026-11-15 02:31:37,668] INFO : Plotted jet ratio +rank0 [2026-11-15 02:31:44,117] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 02:31:44,117] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:32:44,187] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 02:32:44,187] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:33:25,142] INFO : Step 6000/500000 rank0 | Train Loss: 6.7354 | Train Loss Unweighted: 1.4612 | Train Loss Components: Classification: 0.0093 | Classification_binary: 1.2931 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0796 | Regression_eta: 0.0013 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.6920 | Classification_binary: 1.2931 | Regression_cos_phi: 0.4387 | Regression_energy: 0.6327 | Regression_eta: 0.4831 | Regression_pt: 0.6247 | Regression_sin_phi: 2.5710 | LR: 2.19e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4709s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0440 | max_optimizer_step: 0.0621 | max_tensorboard_logging: 0.1281 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2765.7 | uss_MB: 1537.5 | child_rss_MB: 22745.0 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:33:25,144] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 02:33:25,210] INFO : Running validation on rank0 +rank0 [2026-11-15 02:33:25,210] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:33:25,210] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:33:25,328] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 02:33:25,434] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:33:25,534] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:33:25,602] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:33:25,666] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:33:25,730] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:33:25,796] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:33:25,863] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:33:25,971] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:33:26,073] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:33:26,174] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:33:26,241] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:33:26,308] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:33:26,373] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:33:26,440] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:33:26,506] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:33:26,601] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:33:26,606] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:33:26,609] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:33:26,611] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:33:26,612] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:33:26,614] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:33:26,616] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:33:26,617] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:33:26,620] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:33:26,620] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:33:26,622] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:33:26,625] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:33:26,626] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:33:26,628] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:33:26,630] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:33:26,631] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:33:28,093] INFO : VALIDATION | Step=6000/500000 | Train Loss=6.7354 | Valid Loss=4.9036 | Train Loss Unweighted=1.4612 | Valid Loss Unweighted=1.1576 | Stale=0 | ETA=959.4m +rank0 [2026-11-15 02:33:28,093] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:33:28,163] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:33:28,170] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:33:28,214] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:33:28,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:33:28,890] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:33:29,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:33:29,377] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:33:29,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:33:29,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:33:30,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:33:30,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:33:30,562] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:33:30,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:33:31,047] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:33:31,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:33:31,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:33:31,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:33:32,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:33:32,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:33:32,273] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:33:32,279] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:33:32,815] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_6000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:33:33,173] INFO : Plotted number of elements +rank0 [2026-11-15 02:33:34,404] INFO : Plotted jets +rank0 [2026-11-15 02:33:35,735] INFO : Plotted jet ratio +rank0 [2026-11-15 02:33:55,451] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 02:33:55,451] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:34:55,278] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 02:34:55,278] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:35:22,923] INFO : Step 7000/500000 rank0 | Train Loss: 2.1596 | Train Loss Unweighted: 0.7509 | Train Loss Components: Classification: 0.0008 | Classification_binary: 0.6251 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0626 | Regression_eta: 0.0001 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.6251 | Regression_cos_phi: 0.2742 | Regression_energy: 0.4973 | Regression_eta: 0.0497 | Regression_pt: 0.5155 | Regression_sin_phi: 0.1374 | LR: 2.26e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4530s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0586 | max_tensorboard_logging: 0.1228 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2863.7 | uss_MB: 1635.5 | child_rss_MB: 22744.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:35:22,925] INFO : Running validation on rank0 +rank0 [2026-11-15 02:35:22,925] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:35:22,925] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:35:23,043] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 02:35:23,148] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:35:23,250] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:35:23,320] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:35:23,387] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:35:23,453] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:35:23,520] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:35:23,587] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:35:23,696] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:35:23,795] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:35:23,893] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:35:23,958] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:35:24,023] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:35:24,088] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:35:24,153] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:35:24,218] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:35:24,313] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:35:24,318] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:35:24,319] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:35:24,323] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:35:24,325] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:35:24,327] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:35:24,327] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:35:24,329] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:35:24,331] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:35:24,333] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:35:24,335] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:35:24,337] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:35:24,338] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:35:24,339] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:35:24,340] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:35:24,343] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:35:25,747] INFO : VALIDATION | Step=7000/500000 | Train Loss=2.1596 | Valid Loss=4.9072 | Train Loss Unweighted=0.7509 | Valid Loss Unweighted=1.1484 | Stale=1 | ETA=958.8m +rank0 [2026-11-15 02:35:25,747] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:35:25,820] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:35:25,828] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:35:25,872] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:35:26,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:35:26,600] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:35:26,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:35:27,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:35:27,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:35:27,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:35:27,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:35:28,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:35:28,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:35:28,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:35:28,766] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:35:29,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:35:29,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:35:29,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:35:29,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:35:29,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:35:29,971] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:35:29,976] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:35:30,511] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_7000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:35:30,858] INFO : Plotted number of elements +rank0 [2026-11-15 02:35:32,100] INFO : Plotted jets +rank0 [2026-11-15 02:35:33,439] INFO : Plotted jet ratio +rank0 [2026-11-15 02:36:06,291] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 02:36:06,291] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:37:06,192] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 02:37:06,192] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:37:20,639] INFO : Step 8000/500000 rank0 | Train Loss: 6.1202 | Train Loss Unweighted: 0.6884 | Train Loss Components: Classification: 0.0037 | Classification_binary: 0.5405 | Regression_cos_phi: 0.0036 | Regression_energy: 0.0715 | Regression_eta: 0.0006 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.2783 | Classification_binary: 0.5405 | Regression_cos_phi: 2.9209 | Regression_energy: 0.5680 | Regression_eta: 0.2105 | Regression_pt: 0.5626 | Regression_sin_phi: 1.0393 | LR: 2.34e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4758s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0797 | max_tensorboard_logging: 0.1214 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2886.4 | uss_MB: 1658.2 | child_rss_MB: 22742.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:37:20,641] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 02:37:20,707] INFO : Running validation on rank0 +rank0 [2026-11-15 02:37:20,707] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:37:20,707] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:37:20,824] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 02:37:20,929] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:37:21,030] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:37:21,099] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:37:21,163] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:37:21,227] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:37:21,292] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:37:21,357] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:37:21,466] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:37:21,566] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:37:21,666] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:37:21,734] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:37:21,800] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:37:21,865] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:37:21,931] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:37:21,997] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:37:22,096] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:37:22,100] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:37:22,103] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:37:22,106] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:37:22,109] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:37:22,109] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:37:22,111] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:37:22,113] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:37:22,115] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:37:22,117] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:37:22,118] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:37:22,121] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:37:22,122] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:37:22,123] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:37:22,125] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:37:22,128] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:37:23,519] INFO : VALIDATION | Step=8000/500000 | Train Loss=6.1202 | Valid Loss=4.9302 | Train Loss Unweighted=0.6884 | Valid Loss Unweighted=1.1449 | Stale=2 | ETA=957.9m +rank0 [2026-11-15 02:37:23,520] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:37:23,589] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:37:23,596] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:37:23,642] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:37:24,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:37:24,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:37:24,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:37:24,831] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:37:25,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:37:25,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:37:25,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:37:25,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:37:26,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:37:26,253] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:37:26,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:37:26,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:37:26,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:37:27,217] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:37:27,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:37:27,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:37:27,715] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:37:27,720] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:37:28,256] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_8000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:37:28,610] INFO : Plotted number of elements +rank0 [2026-11-15 02:37:29,831] INFO : Plotted jets +rank0 [2026-11-15 02:37:31,165] INFO : Plotted jet ratio +rank0 [2026-11-15 02:38:17,228] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 02:38:17,229] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:39:16,917] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 02:39:16,917] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:39:18,146] INFO : Step 9000/500000 rank0 | Train Loss: 2.1259 | Train Loss Unweighted: 0.7982 | Train Loss Components: Classification: 0.0014 | Classification_binary: 0.7323 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0327 | Regression_eta: 0.0002 | Regression_pt: 0.0308 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.1013 | Classification_binary: 0.7323 | Regression_cos_phi: 0.3060 | Regression_energy: 0.2599 | Regression_eta: 0.0821 | Regression_pt: 0.2574 | Regression_sin_phi: 0.3871 | LR: 2.43e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4641s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0591 | max_tensorboard_logging: 0.1271 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2839.4 | uss_MB: 1611.2 | child_rss_MB: 22735.3 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:39:18,147] INFO : Running validation on rank0 +rank0 [2026-11-15 02:39:18,147] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:39:18,147] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:39:18,265] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 02:39:18,371] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:39:18,471] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:39:18,540] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:39:18,607] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:39:18,673] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:39:18,740] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:39:18,806] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:39:18,917] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:39:19,021] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:39:19,125] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:39:19,191] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:39:19,255] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:39:19,317] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:39:19,381] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:39:19,446] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:39:19,541] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:39:19,547] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:39:19,549] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:39:19,550] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:39:19,553] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:39:19,555] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:39:19,556] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:39:19,558] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:39:19,559] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:39:19,560] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:39:19,563] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:39:19,565] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:39:19,566] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:39:19,568] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:39:19,570] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:39:19,571] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:39:20,961] INFO : VALIDATION | Step=9000/500000 | Train Loss=2.1259 | Valid Loss=4.9481 | Train Loss Unweighted=0.7982 | Valid Loss Unweighted=1.1884 | Stale=3 | ETA=956.5m +rank0 [2026-11-15 02:39:20,961] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:39:21,032] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:39:21,038] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:39:21,084] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:39:21,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:39:21,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:39:22,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:39:22,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:39:22,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:39:22,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:39:22,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:39:23,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:39:23,450] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:39:23,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:39:23,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:39:24,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:39:24,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:39:24,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:39:24,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:39:25,110] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:39:25,160] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:39:25,166] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:39:25,705] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_9000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:39:26,067] INFO : Plotted number of elements +rank0 [2026-11-15 02:39:27,298] INFO : Plotted jets +rank0 [2026-11-15 02:39:28,622] INFO : Plotted jet ratio +rank0 [2026-11-15 02:40:27,929] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 02:40:27,929] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:41:15,683] INFO : Step 10000/500000 rank0 | Train Loss: 2.8813 | Train Loss Unweighted: 0.8393 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.7836 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0259 | Regression_eta: 0.0032 | Regression_pt: 0.0256 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.7836 | Regression_cos_phi: 0.1517 | Regression_energy: 0.2056 | Regression_eta: 1.1859 | Regression_pt: 0.2140 | Regression_sin_phi: 0.3075 | LR: 2.52e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4552s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0583 | max_tensorboard_logging: 0.1246 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2935.5 | uss_MB: 1707.3 | child_rss_MB: 22744.2 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:41:15,685] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 02:41:15,749] INFO : Running validation on rank0 +rank0 [2026-11-15 02:41:15,749] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:41:15,749] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:41:15,865] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 02:41:15,969] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:41:16,072] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:41:16,141] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:41:16,206] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:41:16,272] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:41:16,340] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:41:16,407] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:41:16,520] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:41:16,624] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:41:16,726] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:41:16,792] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:41:16,856] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:41:16,920] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:41:16,985] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:41:17,050] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:41:17,149] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:41:17,155] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:41:17,155] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:41:17,157] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:41:17,159] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:41:17,163] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:41:17,163] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:41:17,165] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:41:17,167] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:41:17,168] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:41:17,171] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:41:17,173] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:41:17,173] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:41:17,174] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:41:17,177] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:41:17,179] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:41:18,571] INFO : VALIDATION | Step=10000/500000 | Train Loss=2.8813 | Valid Loss=5.0090 | Train Loss Unweighted=0.8393 | Valid Loss Unweighted=1.1788 | Stale=4 | ETA=955.2m +rank0 [2026-11-15 02:41:18,572] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:41:18,646] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:41:18,654] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:41:18,700] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:41:19,161] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:41:19,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:41:19,655] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:41:19,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:41:20,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:41:20,373] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:41:20,609] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:41:20,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:41:21,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:41:21,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:41:21,574] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:41:21,827] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:41:22,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:41:22,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:41:22,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:41:22,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:41:22,813] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:41:22,818] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:41:23,357] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:41:23,705] INFO : Plotted number of elements +rank0 [2026-11-15 02:41:24,947] INFO : Plotted jets +rank0 [2026-11-15 02:41:26,297] INFO : Plotted jet ratio +rank0 [2026-11-15 02:41:39,015] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 02:41:39,015] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:42:39,151] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 02:42:39,151] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:43:13,968] INFO : Step 11000/500000 rank0 | Train Loss: 2.7568 | Train Loss Unweighted: 1.3722 | Train Loss Components: Classification: 0.0038 | Classification_binary: 1.3126 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0268 | Regression_eta: 0.0005 | Regression_pt: 0.0279 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.2805 | Classification_binary: 1.3126 | Regression_cos_phi: 0.3194 | Regression_energy: 0.2130 | Regression_eta: 0.1771 | Regression_pt: 0.2334 | Regression_sin_phi: 0.2209 | LR: 2.63e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4863s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0386 | max_optimizer_step: 0.0802 | max_tensorboard_logging: 0.1281 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2962.7 | uss_MB: 1734.5 | child_rss_MB: 22746.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:43:13,970] INFO : Running validation on rank0 +rank0 [2026-11-15 02:43:13,970] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:43:13,970] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:43:14,089] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 02:43:14,193] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:43:14,292] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:43:14,360] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:43:14,426] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:43:14,490] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:43:14,556] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:43:14,621] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:43:14,728] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:43:14,828] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:43:14,927] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:43:14,994] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:43:15,061] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:43:15,127] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:43:15,193] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:43:15,258] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:43:15,355] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:43:15,361] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:43:15,363] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:43:15,365] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:43:15,367] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:43:15,369] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:43:15,370] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:43:15,372] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:43:15,372] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:43:15,374] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:43:15,376] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:43:15,379] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:43:15,380] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:43:15,381] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:43:15,382] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:43:15,386] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:43:16,851] INFO : VALIDATION | Step=11000/500000 | Train Loss=2.7568 | Valid Loss=4.8994 | Train Loss Unweighted=1.3722 | Valid Loss Unweighted=1.1361 | Stale=0 | ETA=954.2m +rank0 [2026-11-15 02:43:16,853] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:43:16,928] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:43:16,935] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:43:16,981] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:43:17,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:43:17,684] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:43:17,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:43:18,176] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:43:18,417] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:43:18,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:43:18,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:43:19,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:43:19,368] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:43:19,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:43:19,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:43:20,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:43:20,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:43:20,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:43:20,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:43:21,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:43:21,062] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:43:21,079] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:43:21,613] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_11000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:43:21,966] INFO : Plotted number of elements +rank0 [2026-11-15 02:43:23,207] INFO : Plotted jets +rank0 [2026-11-15 02:43:24,550] INFO : Plotted jet ratio +rank0 [2026-11-15 02:43:50,523] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 02:43:50,523] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:44:50,533] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 02:44:50,533] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:45:11,975] INFO : Step 12000/500000 rank0 | Train Loss: 1.0232 | Train Loss Unweighted: 0.4469 | Train Loss Components: Classification: 0.0013 | Classification_binary: 0.4117 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0168 | Regression_eta: 0.0002 | Regression_pt: 0.0167 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0959 | Classification_binary: 0.4117 | Regression_cos_phi: 0.1092 | Regression_energy: 0.1337 | Regression_eta: 0.0621 | Regression_pt: 0.1391 | Regression_sin_phi: 0.0714 | LR: 2.75e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4685s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0357 | max_loss: 0.0026 | max_backward: 0.0404 | max_optimizer_step: 0.0598 | max_tensorboard_logging: 0.1246 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2983.6 | uss_MB: 1755.4 | child_rss_MB: 22746.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:45:11,976] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 02:45:12,044] INFO : Running validation on rank0 +rank0 [2026-11-15 02:45:12,044] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:45:12,044] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:45:12,162] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 02:45:12,267] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:45:12,368] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:45:12,435] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:45:12,501] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:45:12,565] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:45:12,631] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:45:12,696] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:45:12,804] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:45:12,905] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:45:13,007] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:45:13,076] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:45:13,143] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:45:13,209] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:45:13,275] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:45:13,341] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:45:13,438] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:45:13,444] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:45:13,446] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:45:13,448] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:45:13,448] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:45:13,452] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:45:13,452] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:45:13,454] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:45:13,456] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:45:13,458] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:45:13,460] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:45:13,462] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:45:13,463] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:45:13,463] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:45:13,466] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:45:13,467] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:45:14,858] INFO : VALIDATION | Step=12000/500000 | Train Loss=1.0232 | Valid Loss=5.2057 | Train Loss Unweighted=0.4469 | Valid Loss Unweighted=1.2596 | Stale=1 | ETA=952.9m +rank0 [2026-11-15 02:45:14,858] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:45:14,928] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:45:14,935] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:45:14,982] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:45:15,438] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:45:15,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:45:15,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:45:16,173] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:45:16,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:45:16,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:45:16,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:45:17,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:45:17,393] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:45:17,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:45:17,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:45:18,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:45:18,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:45:18,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:45:18,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:45:19,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:45:19,124] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:45:19,130] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:45:19,667] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_12000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:45:20,023] INFO : Plotted number of elements +rank0 [2026-11-15 02:45:21,253] INFO : Plotted jets +rank0 [2026-11-15 02:45:22,586] INFO : Plotted jet ratio +rank0 [2026-11-15 02:46:01,778] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 02:46:01,778] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:47:01,584] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 02:47:01,584] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:47:09,812] INFO : Step 13000/500000 rank0 | Train Loss: 2.7919 | Train Loss Unweighted: 0.9411 | Train Loss Components: Classification: 0.0025 | Classification_binary: 0.7659 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0833 | Regression_eta: 0.0005 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.1856 | Classification_binary: 0.7659 | Regression_cos_phi: 0.1062 | Regression_energy: 0.6622 | Regression_eta: 0.1931 | Regression_pt: 0.7399 | Regression_sin_phi: 0.1391 | LR: 2.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4604s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0028 | max_backward: 0.0391 | max_optimizer_step: 0.0580 | max_tensorboard_logging: 0.1248 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3016.7 | uss_MB: 1788.5 | child_rss_MB: 22746.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:47:09,814] INFO : Running validation on rank0 +rank0 [2026-11-15 02:47:09,814] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:47:09,814] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:47:09,932] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 02:47:10,037] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:47:10,138] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:47:10,207] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:47:10,274] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:47:10,340] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:47:10,407] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:47:10,474] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:47:10,585] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:47:10,686] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:47:10,784] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:47:10,850] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:47:10,915] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:47:10,978] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:47:11,043] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:47:11,109] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:47:11,204] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:47:11,210] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:47:11,210] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:47:11,213] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:47:11,216] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:47:11,216] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:47:11,218] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:47:11,220] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:47:11,222] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:47:11,224] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:47:11,226] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:47:11,228] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:47:11,229] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:47:11,230] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:47:11,232] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:47:11,235] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:47:12,623] INFO : VALIDATION | Step=13000/500000 | Train Loss=2.7919 | Valid Loss=5.2815 | Train Loss Unweighted=0.9411 | Valid Loss Unweighted=1.2289 | Stale=2 | ETA=951.3m +rank0 [2026-11-15 02:47:12,623] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:47:12,696] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:47:12,702] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:47:12,748] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:47:13,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:47:13,457] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:47:13,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:47:13,947] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:47:14,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:47:14,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:47:14,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:47:14,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:47:15,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:47:15,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:47:15,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:47:15,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:47:16,074] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:47:16,304] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:47:16,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:47:16,755] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:47:16,808] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:47:16,813] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:47:17,346] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_13000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:47:17,684] INFO : Plotted number of elements +rank0 [2026-11-15 02:47:18,907] INFO : Plotted jets +rank0 [2026-11-15 02:47:20,226] INFO : Plotted jet ratio +rank0 [2026-11-15 02:48:11,857] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 02:48:11,857] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:49:06,018] INFO : Step 14000/500000 rank0 | Train Loss: 2.8806 | Train Loss Unweighted: 0.9824 | Train Loss Components: Classification: 0.0018 | Classification_binary: 0.8855 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0469 | Regression_eta: 0.0015 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.1358 | Classification_binary: 0.8855 | Regression_cos_phi: 0.2221 | Regression_energy: 0.3729 | Regression_eta: 0.5385 | Regression_pt: 0.3845 | Regression_sin_phi: 0.3413 | LR: 3.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4647s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0595 | max_tensorboard_logging: 0.1283 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3032.1 | uss_MB: 1803.9 | child_rss_MB: 22746.4 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:49:06,023] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 02:49:06,089] INFO : Running validation on rank0 +rank0 [2026-11-15 02:49:06,089] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:49:06,089] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:49:06,205] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 02:49:06,312] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:49:06,414] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:49:06,482] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:49:06,548] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:49:06,615] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:49:06,682] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:49:06,750] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:49:06,861] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:49:06,962] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:49:07,063] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:49:07,131] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:49:07,195] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:49:07,259] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:49:07,323] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:49:07,387] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:49:07,483] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:49:07,489] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:49:07,491] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:49:07,492] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:49:07,495] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:49:07,495] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:49:07,497] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:49:07,499] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:49:07,501] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:49:07,503] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:49:07,505] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:49:07,507] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:49:07,508] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:49:07,509] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:49:07,511] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:49:07,514] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:49:08,895] INFO : VALIDATION | Step=14000/500000 | Train Loss=2.8806 | Valid Loss=5.3833 | Train Loss Unweighted=0.9824 | Valid Loss Unweighted=1.2278 | Stale=3 | ETA=948.8m +rank0 [2026-11-15 02:49:08,895] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:49:08,968] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:49:08,975] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:49:09,021] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:49:09,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:49:09,752] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:49:09,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:49:10,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:49:10,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:49:10,716] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:49:10,953] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:49:11,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:49:11,453] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:49:11,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:49:11,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:49:12,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:49:12,421] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:49:12,659] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:49:12,899] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:49:13,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:49:13,170] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:49:13,175] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:49:13,708] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_14000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:49:14,062] INFO : Plotted number of elements +rank0 [2026-11-15 02:49:15,299] INFO : Plotted jets +rank0 [2026-11-15 02:49:16,634] INFO : Plotted jet ratio +rank0 [2026-11-15 02:49:22,270] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 02:49:22,270] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:50:21,811] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 02:50:21,811] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:51:03,307] INFO : Step 15000/500000 rank0 | Train Loss: 2.7781 | Train Loss Unweighted: 1.2876 | Train Loss Components: Classification: 0.0044 | Classification_binary: 1.2138 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0292 | Regression_eta: 0.0007 | Regression_pt: 0.0392 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.3266 | Classification_binary: 1.2138 | Regression_cos_phi: 0.1044 | Regression_energy: 0.2319 | Regression_eta: 0.2555 | Regression_pt: 0.3272 | Regression_sin_phi: 0.3187 | LR: 3.17e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4656s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0394 | max_optimizer_step: 0.0626 | max_tensorboard_logging: 0.1207 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3058.7 | uss_MB: 1830.5 | child_rss_MB: 22746.2 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:51:03,308] INFO : Running validation on rank0 +rank0 [2026-11-15 02:51:03,308] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:51:03,308] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:51:03,425] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 02:51:03,531] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:51:03,631] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:51:03,700] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:51:03,766] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:51:03,834] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:51:03,903] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:51:03,970] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:51:04,082] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:51:04,185] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:51:04,285] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:51:04,352] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:51:04,416] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:51:04,480] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:51:04,544] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:51:04,608] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:51:04,706] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:51:04,712] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:51:04,714] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:51:04,715] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:51:04,718] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:51:04,720] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:51:04,721] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:51:04,723] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:51:04,724] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:51:04,727] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:51:04,728] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:51:04,730] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:51:04,731] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:51:04,733] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:51:04,734] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:51:04,737] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:51:06,133] INFO : VALIDATION | Step=15000/500000 | Train Loss=2.7781 | Valid Loss=5.4673 | Train Loss Unweighted=1.2876 | Valid Loss Unweighted=1.2550 | Stale=4 | ETA=946.9m +rank0 [2026-11-15 02:51:06,133] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:51:06,205] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:51:06,211] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:51:06,258] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:51:06,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:51:06,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:51:07,221] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:51:07,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:51:07,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:51:07,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:51:08,178] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:51:08,415] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:51:08,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:51:08,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:51:09,145] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:51:09,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:51:09,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:51:09,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:51:10,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:51:10,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:51:10,362] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:51:10,367] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:51:10,903] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_15000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:51:11,250] INFO : Plotted number of elements +rank0 [2026-11-15 02:51:12,485] INFO : Plotted jets +rank0 [2026-11-15 02:51:13,840] INFO : Plotted jet ratio +rank0 [2026-11-15 02:51:32,739] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 02:51:32,739] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:52:32,556] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 02:52:32,556] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:53:00,864] INFO : Step 16000/500000 rank0 | Train Loss: 2.0774 | Train Loss Unweighted: 0.5840 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.5179 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0316 | Regression_eta: 0.0004 | Regression_pt: 0.0327 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0332 | Classification_binary: 0.5179 | Regression_cos_phi: 0.5938 | Regression_energy: 0.2512 | Regression_eta: 0.1317 | Regression_pt: 0.2729 | Regression_sin_phi: 0.2768 | LR: 3.33e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4663s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0387 | max_optimizer_step: 0.0609 | max_tensorboard_logging: 0.1224 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3088.9 | uss_MB: 1860.7 | child_rss_MB: 22743.7 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:53:00,866] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 02:53:00,933] INFO : Running validation on rank0 +rank0 [2026-11-15 02:53:00,933] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:53:00,933] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:53:01,052] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 02:53:01,159] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:53:01,261] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:53:01,330] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:53:01,396] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:53:01,462] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:53:01,531] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:53:01,599] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:53:01,710] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:53:01,813] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:53:01,912] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:53:01,979] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:53:02,046] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:53:02,112] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:53:02,177] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:53:02,242] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:53:02,340] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:53:02,346] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:53:02,348] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:53:02,350] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:53:02,352] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:53:02,354] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:53:02,355] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:53:02,356] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:53:02,357] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:53:02,360] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:53:02,363] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:53:02,365] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:53:02,365] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:53:02,367] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:53:02,369] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:53:02,371] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:53:03,760] INFO : VALIDATION | Step=16000/500000 | Train Loss=2.0774 | Valid Loss=5.4978 | Train Loss Unweighted=0.5840 | Valid Loss Unweighted=1.3049 | Stale=5 | ETA=945.2m +rank0 [2026-11-15 02:53:03,760] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:53:03,832] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:53:03,839] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:53:03,886] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:53:04,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:53:04,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:53:04,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:53:05,113] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:53:05,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:53:05,584] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:53:05,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:53:06,070] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:53:06,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:53:06,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:53:06,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:53:07,027] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:53:07,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:53:07,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:53:07,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:53:07,960] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:53:08,006] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:53:08,012] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:53:08,546] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_16000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:53:08,890] INFO : Plotted number of elements +rank0 [2026-11-15 02:53:10,131] INFO : Plotted jets +rank0 [2026-11-15 02:53:11,517] INFO : Plotted jet ratio +rank0 [2026-11-15 02:53:43,504] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 02:53:43,504] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:54:42,645] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 02:54:42,645] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:54:57,862] INFO : Step 17000/500000 rank0 | Train Loss: 1.1839 | Train Loss Unweighted: 0.3385 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.2588 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0396 | Regression_eta: 0.0001 | Regression_pt: 0.0395 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0153 | Classification_binary: 0.2588 | Regression_cos_phi: 0.0858 | Regression_energy: 0.3143 | Regression_eta: 0.0517 | Regression_pt: 0.3300 | Regression_sin_phi: 0.1280 | LR: 3.51e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4707s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0602 | max_tensorboard_logging: 0.1243 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3105.0 | uss_MB: 1876.8 | child_rss_MB: 22739.1 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:54:57,863] INFO : Running validation on rank0 +rank0 [2026-11-15 02:54:57,863] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:54:57,863] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:54:57,981] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 02:54:58,090] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:54:58,193] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:54:58,260] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:54:58,327] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:54:58,393] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:54:58,460] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:54:58,528] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:54:58,637] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:54:58,737] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:54:58,837] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:54:58,903] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:54:58,967] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:54:59,032] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:54:59,099] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:54:59,163] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:54:59,259] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:54:59,265] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:54:59,268] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:54:59,268] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:54:59,270] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:54:59,273] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:54:59,274] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:54:59,276] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:54:59,278] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:54:59,280] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:54:59,281] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:54:59,282] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:54:59,284] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:54:59,286] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:54:59,288] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:54:59,290] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:55:00,662] INFO : VALIDATION | Step=17000/500000 | Train Loss=1.1839 | Valid Loss=5.5486 | Train Loss Unweighted=0.3385 | Valid Loss Unweighted=1.3199 | Stale=6 | ETA=943.1m +rank0 [2026-11-15 02:55:00,663] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:55:00,735] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:55:00,741] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:55:00,786] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:55:01,254] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:55:01,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:55:01,741] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:55:01,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:55:02,221] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:55:02,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:55:02,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:55:02,932] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:55:03,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:55:03,418] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:55:03,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:55:03,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:55:04,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:55:04,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:55:04,603] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:55:04,819] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:55:04,875] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:55:04,881] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:55:05,417] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_17000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:55:05,760] INFO : Plotted number of elements +rank0 [2026-11-15 02:55:06,994] INFO : Plotted jets +rank0 [2026-11-15 02:55:08,346] INFO : Plotted jet ratio +rank0 [2026-11-15 02:55:53,478] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 02:55:53,478] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:56:52,748] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 02:56:52,748] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:56:54,824] INFO : Step 18000/500000 rank0 | Train Loss: 1.5685 | Train Loss Unweighted: 0.5531 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.4800 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0361 | Regression_eta: 0.0003 | Regression_pt: 0.0360 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0145 | Classification_binary: 0.4800 | Regression_cos_phi: 0.0981 | Regression_energy: 0.2869 | Regression_eta: 0.1241 | Regression_pt: 0.3010 | Regression_sin_phi: 0.2639 | LR: 3.69e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4622s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0388 | max_optimizer_step: 0.0602 | max_tensorboard_logging: 0.1208 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3130.2 | uss_MB: 1902.0 | child_rss_MB: 22745.1 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:56:54,826] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 02:56:54,889] INFO : Running validation on rank0 +rank0 [2026-11-15 02:56:54,889] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:56:54,889] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:56:55,018] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 02:56:55,125] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:56:55,228] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:56:55,297] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:56:55,363] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:56:55,429] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:56:55,497] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:56:55,566] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:56:55,676] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:56:55,778] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:56:55,879] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:56:55,945] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:56:56,011] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:56:56,077] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:56:56,142] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:56:56,207] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:56:56,305] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:56:56,310] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:56:56,313] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:56:56,316] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:56:56,316] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:56:56,319] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:56:56,320] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:56:56,322] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:56:56,324] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:56:56,326] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:56:56,327] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:56:56,330] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:56:56,330] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:56:56,332] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:56:56,334] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:56:56,335] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:56:57,730] INFO : VALIDATION | Step=18000/500000 | Train Loss=1.5685 | Valid Loss=5.8648 | Train Loss Unweighted=0.5531 | Valid Loss Unweighted=1.3450 | Stale=7 | ETA=941.1m +rank0 [2026-11-15 02:56:57,731] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:56:57,802] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:56:57,809] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:56:57,856] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:56:58,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:56:58,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:56:58,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:56:59,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:56:59,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:56:59,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:56:59,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:57:00,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:57:00,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:57:00,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:57:00,725] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:57:00,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:57:01,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:57:01,429] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:57:01,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:57:01,881] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:57:01,936] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:57:01,942] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:57:02,480] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_18000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:57:02,820] INFO : Plotted number of elements +rank0 [2026-11-15 02:57:04,047] INFO : Plotted jets +rank0 [2026-11-15 02:57:05,384] INFO : Plotted jet ratio +rank0 [2026-11-15 02:58:03,335] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 02:58:03,335] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:58:51,581] INFO : Step 19000/500000 rank0 | Train Loss: 1.9427 | Train Loss Unweighted: 0.7975 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.7566 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0198 | Regression_eta: 0.0002 | Regression_pt: 0.0195 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0320 | Classification_binary: 0.7566 | Regression_cos_phi: 0.5493 | Regression_energy: 0.1572 | Regression_eta: 0.0896 | Regression_pt: 0.1633 | Regression_sin_phi: 0.1947 | LR: 3.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4772s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0026 | max_backward: 0.0431 | max_optimizer_step: 0.0635 | max_tensorboard_logging: 0.1213 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3162.4 | uss_MB: 1934.3 | child_rss_MB: 22745.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:58:51,582] INFO : Running validation on rank0 +rank0 [2026-11-15 02:58:51,582] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:58:51,582] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:58:51,700] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 02:58:51,803] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:58:51,904] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:58:51,971] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:58:52,036] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:58:52,102] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:58:52,167] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:58:52,233] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:58:52,341] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:58:52,441] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:58:52,542] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:58:52,610] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:58:52,676] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:58:52,741] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:58:52,808] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:58:52,874] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:58:52,975] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:58:52,979] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:58:52,983] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:58:52,984] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:58:52,987] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:58:52,988] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:58:52,990] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:58:52,990] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:58:52,993] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:58:52,995] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:58:52,997] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:58:52,999] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:58:53,000] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:58:53,000] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:58:53,002] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:58:53,006] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:58:54,387] INFO : VALIDATION | Step=19000/500000 | Train Loss=1.9427 | Valid Loss=6.0428 | Train Loss Unweighted=0.7975 | Valid Loss Unweighted=1.4201 | Stale=8 | ETA=939.0m +rank0 [2026-11-15 02:58:54,388] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:58:54,457] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:58:54,463] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:58:54,509] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:58:54,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:58:55,198] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:58:55,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:58:55,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:58:55,919] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:58:56,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:58:56,404] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:58:56,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:58:56,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:58:57,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:58:57,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:58:57,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:58:57,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:58:58,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:58:58,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:58:58,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:58:58,572] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:58:58,579] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:58:59,103] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_19000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:58:59,454] INFO : Plotted number of elements +rank0 [2026-11-15 02:59:00,675] INFO : Plotted jets +rank0 [2026-11-15 02:59:02,004] INFO : Plotted jet ratio +rank0 [2026-11-15 02:59:13,692] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 02:59:13,692] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:00:12,894] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 03:00:12,895] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:00:48,053] INFO : Step 20000/500000 rank0 | Train Loss: 0.6223 | Train Loss Unweighted: 0.1651 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.1300 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0170 | Regression_eta: 0.0001 | Regression_pt: 0.0177 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0130 | Classification_binary: 0.1300 | Regression_cos_phi: 0.0804 | Regression_energy: 0.1348 | Regression_eta: 0.0442 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0722 | LR: 4.07e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4933s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0415 | max_optimizer_step: 0.0620 | max_tensorboard_logging: 0.1259 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3110.5 | uss_MB: 1882.3 | child_rss_MB: 22747.8 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:00:48,055] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 03:00:48,121] INFO : Running validation on rank0 +rank0 [2026-11-15 03:00:48,121] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:00:48,121] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:00:48,238] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 03:00:48,345] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:00:48,445] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:00:48,512] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:00:48,577] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:00:48,641] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:00:48,706] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:00:48,771] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:00:48,882] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:00:48,986] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:00:49,091] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:00:49,158] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:00:49,225] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:00:49,292] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:00:49,359] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:00:49,426] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:00:49,527] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:00:49,531] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:00:49,535] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:00:49,537] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:00:49,539] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:00:49,541] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:00:49,542] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:00:49,544] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:00:49,544] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:00:49,546] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:00:49,549] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:00:49,551] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:00:49,552] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:00:49,554] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:00:49,554] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:00:49,558] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:00:50,955] INFO : VALIDATION | Step=20000/500000 | Train Loss=0.6223 | Valid Loss=6.2160 | Train Loss Unweighted=0.1651 | Valid Loss Unweighted=1.4715 | Stale=9 | ETA=936.8m +rank0 [2026-11-15 03:00:50,956] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:00:51,027] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:00:51,034] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:00:51,080] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:00:51,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:00:51,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:00:52,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:00:52,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:00:52,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:00:52,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:00:52,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:00:53,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:00:53,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:00:53,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:00:53,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:00:54,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:00:54,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:00:54,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:00:54,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:00:55,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:00:55,179] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:00:55,184] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:00:55,715] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:00:56,071] INFO : Plotted number of elements +rank0 [2026-11-15 03:00:57,315] INFO : Plotted jets +rank0 [2026-11-15 03:00:58,656] INFO : Plotted jet ratio +rank0 [2026-11-15 03:01:23,636] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 03:01:23,636] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:02:23,263] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 03:02:23,264] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:02:45,480] INFO : Step 21000/500000 rank0 | Train Loss: 1.2808 | Train Loss Unweighted: 0.4894 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.4064 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0413 | Regression_eta: 0.0002 | Regression_pt: 0.0414 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0003 | Classification_binary: 0.4064 | Regression_cos_phi: 0.0511 | Regression_energy: 0.3282 | Regression_eta: 0.0681 | Regression_pt: 0.3454 | Regression_sin_phi: 0.0814 | LR: 4.28e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4618s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0591 | max_tensorboard_logging: 0.1252 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3214.5 | uss_MB: 1986.3 | child_rss_MB: 22747.1 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:02:45,481] INFO : Running validation on rank0 +rank0 [2026-11-15 03:02:45,481] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:02:45,481] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:02:45,598] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 03:02:45,701] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:02:45,800] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:02:45,869] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:02:45,935] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:02:46,000] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:02:46,068] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:02:46,136] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:02:46,246] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:02:46,347] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:02:46,445] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:02:46,511] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:02:46,574] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:02:46,637] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:02:46,701] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:02:46,765] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:02:46,863] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:02:46,868] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:02:46,870] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:02:46,874] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:02:46,875] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:02:46,876] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:02:46,878] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:02:46,880] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:02:46,882] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:02:46,883] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:02:46,886] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:02:46,886] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:02:46,888] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:02:46,890] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:02:46,892] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:02:46,894] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:02:48,278] INFO : VALIDATION | Step=21000/500000 | Train Loss=1.2808 | Valid Loss=6.2011 | Train Loss Unweighted=0.4894 | Valid Loss Unweighted=1.5632 | Stale=10 | ETA=934.9m +rank0 [2026-11-15 03:02:48,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:02:48,350] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:02:48,356] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:02:48,402] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:02:48,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:02:49,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:02:49,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:02:49,601] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:02:49,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:02:50,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:02:50,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:02:50,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:02:50,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:02:51,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:02:51,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:02:51,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:02:51,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:02:51,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:02:52,222] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:02:52,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:02:52,494] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:02:52,499] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:02:53,027] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_21000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:02:53,369] INFO : Plotted number of elements +rank0 [2026-11-15 03:02:54,596] INFO : Plotted jets +rank0 [2026-11-15 03:02:55,923] INFO : Plotted jet ratio +rank0 [2026-11-15 03:03:33,825] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 03:03:33,825] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:04:33,425] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 03:04:33,425] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:04:42,523] INFO : Step 22000/500000 rank0 | Train Loss: 1.6328 | Train Loss Unweighted: 0.8123 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.7170 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0472 | Regression_eta: 0.0001 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0005 | Classification_binary: 0.7170 | Regression_cos_phi: 0.0443 | Regression_energy: 0.3749 | Regression_eta: 0.0381 | Regression_pt: 0.3998 | Regression_sin_phi: 0.0583 | LR: 4.50e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4770s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0032 | max_backward: 0.0399 | max_optimizer_step: 0.0594 | max_tensorboard_logging: 0.1233 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3237.8 | uss_MB: 2009.6 | child_rss_MB: 22746.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:04:42,525] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 03:04:42,590] INFO : Running validation on rank0 +rank0 [2026-11-15 03:04:42,590] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:04:42,590] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:04:42,706] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 03:04:42,808] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:04:42,907] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:04:42,974] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:04:43,039] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:04:43,103] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:04:43,169] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:04:43,234] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:04:43,343] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:04:43,445] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:04:43,546] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:04:43,613] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:04:43,679] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:04:43,744] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:04:43,810] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:04:43,877] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:04:43,977] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:04:43,983] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:04:43,984] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:04:43,986] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:04:43,989] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:04:43,990] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:04:43,991] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:04:43,992] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:04:43,994] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:04:43,997] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:04:43,999] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:04:44,001] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:04:44,002] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:04:44,002] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:04:44,006] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:04:44,006] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:04:45,385] INFO : VALIDATION | Step=22000/500000 | Train Loss=1.6328 | Valid Loss=6.4909 | Train Loss Unweighted=0.8123 | Valid Loss Unweighted=1.5919 | Stale=11 | ETA=933.0m +rank0 [2026-11-15 03:04:45,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:04:45,458] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:04:45,465] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:04:45,511] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:04:45,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:04:46,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:04:46,473] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:04:46,712] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:04:46,951] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:04:47,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:04:47,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:04:47,659] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:04:47,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:04:48,132] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:04:48,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:04:48,614] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:04:48,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:04:49,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:04:49,323] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:04:49,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:04:49,598] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:04:49,604] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:04:50,137] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_22000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:04:50,491] INFO : Plotted number of elements +rank0 [2026-11-15 03:04:51,706] INFO : Plotted jets +rank0 [2026-11-15 03:04:53,042] INFO : Plotted jet ratio +rank0 [2026-11-15 03:05:44,489] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 03:05:44,489] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:06:40,045] INFO : Step 23000/500000 rank0 | Train Loss: 1.3924 | Train Loss Unweighted: 0.6716 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.6249 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0226 | Regression_eta: 0.0003 | Regression_pt: 0.0232 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0347 | Classification_binary: 0.6249 | Regression_cos_phi: 0.1225 | Regression_energy: 0.1793 | Regression_eta: 0.1106 | Regression_pt: 0.1935 | Regression_sin_phi: 0.1270 | LR: 4.73e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4515s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0345 | max_loss: 0.0025 | max_backward: 0.0382 | max_optimizer_step: 0.0586 | max_tensorboard_logging: 0.1150 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3249.0 | uss_MB: 2020.9 | child_rss_MB: 22745.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:06:40,047] INFO : Running validation on rank0 +rank0 [2026-11-15 03:06:40,047] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:06:40,047] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:06:40,165] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 03:06:40,269] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:06:40,371] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:06:40,438] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:06:40,505] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:06:40,572] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:06:40,639] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:06:40,707] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:06:40,816] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:06:40,915] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:06:41,013] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:06:41,081] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:06:41,147] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:06:41,210] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:06:41,275] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:06:41,339] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:06:41,441] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:06:41,447] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:06:41,449] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:06:41,451] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:06:41,453] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:06:41,455] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:06:41,456] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:06:41,458] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:06:41,460] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:06:41,460] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:06:41,462] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:06:41,464] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:06:41,466] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:06:41,467] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:06:41,468] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:06:41,470] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:06:42,853] INFO : VALIDATION | Step=23000/500000 | Train Loss=1.3924 | Valid Loss=6.4271 | Train Loss Unweighted=0.6716 | Valid Loss Unweighted=1.6276 | Stale=12 | ETA=931.2m +rank0 [2026-11-15 03:06:42,854] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:06:42,922] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:06:42,928] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:06:42,974] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:06:43,448] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:06:43,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:06:43,944] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:06:44,186] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:06:44,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:06:44,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:06:44,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:06:45,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:06:45,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:06:45,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:06:45,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:06:46,121] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:06:46,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:06:46,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:06:46,830] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:06:47,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:06:47,125] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:06:47,130] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:06:47,660] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_23000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:06:48,012] INFO : Plotted number of elements +rank0 [2026-11-15 03:06:49,252] INFO : Plotted jets +rank0 [2026-11-15 03:06:50,600] INFO : Plotted jet ratio +rank0 [2026-11-15 03:06:55,352] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 03:06:55,352] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:07:55,171] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 03:07:55,171] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:08:37,399] INFO : Step 24000/500000 rank0 | Train Loss: 2.1361 | Train Loss Unweighted: 0.7894 | Train Loss Components: Classification: 0.0011 | Classification_binary: 0.7283 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0264 | Regression_eta: 0.0003 | Regression_pt: 0.0324 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0803 | Classification_binary: 0.7283 | Regression_cos_phi: 0.3155 | Regression_energy: 0.2101 | Regression_eta: 0.1175 | Regression_pt: 0.2707 | Regression_sin_phi: 0.4138 | LR: 4.97e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4557s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0026 | max_backward: 0.0385 | max_optimizer_step: 0.0599 | max_tensorboard_logging: 0.1147 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3283.7 | uss_MB: 2055.5 | child_rss_MB: 22745.4 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:08:37,401] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 03:08:37,466] INFO : Running validation on rank0 +rank0 [2026-11-15 03:08:37,466] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:08:37,466] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:08:37,582] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 03:08:37,684] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:08:37,782] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:08:37,849] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:08:37,915] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:08:37,980] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:08:38,048] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:08:38,114] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:08:38,226] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:08:38,329] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:08:38,430] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:08:38,496] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:08:38,560] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:08:38,624] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:08:38,688] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:08:38,751] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:08:38,850] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:08:38,854] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:08:38,856] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:08:38,860] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:08:38,861] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:08:38,863] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:08:38,864] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:08:38,866] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:08:38,868] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:08:38,870] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:08:38,870] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:08:38,872] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:08:38,874] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:08:38,876] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:08:38,877] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:08:38,880] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:08:40,254] INFO : VALIDATION | Step=24000/500000 | Train Loss=2.1361 | Valid Loss=6.7511 | Train Loss Unweighted=0.7894 | Valid Loss Unweighted=1.7583 | Stale=13 | ETA=929.3m +rank0 [2026-11-15 03:08:40,254] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:08:40,324] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:08:40,331] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:08:40,378] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:08:40,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:08:41,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:08:41,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:08:41,566] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:08:41,808] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:08:42,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:08:42,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:08:42,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:08:42,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:08:42,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:08:43,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:08:43,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:08:43,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:08:43,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:08:44,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:08:44,408] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:08:44,464] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:08:44,469] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:08:45,006] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_24000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:08:45,357] INFO : Plotted number of elements +rank0 [2026-11-15 03:08:46,584] INFO : Plotted jets +rank0 [2026-11-15 03:08:47,928] INFO : Plotted jet ratio +rank0 [2026-11-15 03:09:05,941] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 03:09:05,941] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:10:05,762] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 03:10:05,763] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:10:35,162] INFO : Step 25000/500000 rank0 | Train Loss: 0.6121 | Train Loss Unweighted: 0.1180 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0914 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0131 | Regression_eta: 0.0001 | Regression_pt: 0.0131 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0004 | Classification_binary: 0.0914 | Regression_cos_phi: 0.1352 | Regression_energy: 0.1037 | Regression_eta: 0.0520 | Regression_pt: 0.1091 | Regression_sin_phi: 0.1204 | LR: 5.22e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5664s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0024 | max_backward: 0.0380 | max_optimizer_step: 0.0586 | max_tensorboard_logging: 0.2311 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3310.6 | uss_MB: 2082.5 | child_rss_MB: 22742.9 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:10:35,164] INFO : Running validation on rank0 +rank0 [2026-11-15 03:10:35,164] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:10:35,164] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:10:35,292] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 03:10:35,397] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:10:35,503] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:10:35,571] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:10:35,638] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:10:35,703] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:10:35,769] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:10:35,835] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:10:35,947] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:10:36,049] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:10:36,150] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:10:36,216] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:10:36,282] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:10:36,346] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:10:36,411] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:10:36,475] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:10:36,580] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:10:36,586] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:10:36,587] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:10:36,589] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:10:36,592] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:10:36,594] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:10:36,595] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:10:36,597] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:10:36,598] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:10:36,601] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:10:36,602] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:10:36,604] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:10:36,605] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:10:36,607] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:10:36,608] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:10:36,611] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:10:38,012] INFO : VALIDATION | Step=25000/500000 | Train Loss=0.6121 | Valid Loss=6.7883 | Train Loss Unweighted=0.1180 | Valid Loss Unweighted=1.8468 | Stale=14 | ETA=927.5m +rank0 [2026-11-15 03:10:38,012] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:10:38,130] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:10:38,137] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:10:38,183] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:10:38,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:10:38,900] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:10:39,137] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:10:39,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:10:39,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:10:39,865] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:10:40,110] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:10:40,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:10:40,586] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:10:40,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:10:41,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:10:41,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:10:41,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:10:41,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:10:42,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:10:42,246] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:10:42,297] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:10:42,304] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:10:42,836] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_25000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:10:43,192] INFO : Plotted number of elements +rank0 [2026-11-15 03:10:44,440] INFO : Plotted jets +rank0 [2026-11-15 03:10:45,786] INFO : Plotted jet ratio +rank0 [2026-11-15 03:11:17,030] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 03:11:17,030] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:12:17,013] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 03:12:17,013] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:12:33,164] INFO : Step 26000/500000 rank0 | Train Loss: 0.5959 | Train Loss Unweighted: 0.0692 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0235 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0232 | Regression_eta: 0.0001 | Regression_pt: 0.0221 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0007 | Classification_binary: 0.0235 | Regression_cos_phi: 0.0732 | Regression_energy: 0.1847 | Regression_eta: 0.0532 | Regression_pt: 0.1847 | Regression_sin_phi: 0.0760 | LR: 5.47e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4839s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0730 | max_tensorboard_logging: 0.1280 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3319.9 | uss_MB: 2091.8 | child_rss_MB: 22742.9 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:12:33,166] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 03:12:33,234] INFO : Running validation on rank0 +rank0 [2026-11-15 03:12:33,234] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:12:33,234] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:12:33,354] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 03:12:33,458] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:12:33,558] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:12:33,624] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:12:33,689] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:12:33,753] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:12:33,820] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:12:33,886] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:12:33,997] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:12:34,100] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:12:34,200] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:12:34,268] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:12:34,333] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:12:34,399] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:12:34,466] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:12:34,531] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:12:34,637] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:12:34,642] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:12:34,645] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:12:34,648] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:12:34,650] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:12:34,651] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:12:34,652] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:12:34,654] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:12:34,656] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:12:34,658] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:12:34,660] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:12:34,661] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:12:34,662] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:12:34,663] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:12:34,666] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:12:34,668] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:12:36,064] INFO : VALIDATION | Step=26000/500000 | Train Loss=0.5959 | Valid Loss=6.7923 | Train Loss Unweighted=0.0692 | Valid Loss Unweighted=1.8443 | Stale=15 | ETA=925.9m +rank0 [2026-11-15 03:12:36,065] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:12:36,136] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:12:36,144] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:12:36,192] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:12:36,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:12:36,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:12:37,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:12:37,382] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:12:37,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:12:37,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:12:38,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:12:38,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:12:38,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:12:38,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:12:39,056] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:12:39,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:12:39,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:12:39,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:12:40,023] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:12:40,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:12:40,299] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:12:40,304] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:12:40,837] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_26000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:12:41,194] INFO : Plotted number of elements +rank0 [2026-11-15 03:12:42,421] INFO : Plotted jets +rank0 [2026-11-15 03:12:43,762] INFO : Plotted jet ratio +rank0 [2026-11-15 03:13:28,189] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 03:13:28,189] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:14:28,082] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 03:14:28,082] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:14:31,025] INFO : Step 27000/500000 rank0 | Train Loss: 1.1721 | Train Loss Unweighted: 0.2035 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0242 | Regression_eta: 0.0006 | Regression_pt: 0.0242 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0061 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0482 | Regression_energy: 0.1923 | Regression_eta: 0.2290 | Regression_pt: 0.2018 | Regression_sin_phi: 0.3408 | LR: 5.74e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4751s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0025 | max_backward: 0.0386 | max_optimizer_step: 0.0609 | max_tensorboard_logging: 0.1256 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3352.8 | uss_MB: 2124.6 | child_rss_MB: 22746.7 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:14:31,027] INFO : Running validation on rank0 +rank0 [2026-11-15 03:14:31,027] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:14:31,027] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:14:31,150] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 03:14:31,253] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:14:31,354] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:14:31,422] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:14:31,487] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:14:31,552] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:14:31,618] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:14:31,684] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:14:31,795] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:14:31,895] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:14:31,994] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:14:32,063] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:14:32,129] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:14:32,196] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:14:32,262] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:14:32,328] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:14:32,436] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:14:32,443] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:14:32,445] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:14:32,447] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:14:32,448] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:14:32,451] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:14:32,451] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:14:32,453] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:14:32,455] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:14:32,457] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:14:32,459] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:14:32,461] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:14:32,462] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:14:32,463] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:14:32,464] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:14:32,466] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:14:33,877] INFO : VALIDATION | Step=27000/500000 | Train Loss=1.1721 | Valid Loss=7.2023 | Train Loss Unweighted=0.2035 | Valid Loss Unweighted=1.8696 | Stale=16 | ETA=924.1m +rank0 [2026-11-15 03:14:33,877] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:14:33,949] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:14:33,955] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:14:34,004] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:14:34,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:14:34,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:14:34,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:14:35,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:14:35,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:14:35,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:14:35,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:14:36,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:14:36,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:14:36,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:14:36,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:14:37,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:14:37,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:14:37,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:14:37,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:14:38,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:14:38,095] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:14:38,101] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:14:38,629] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_27000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:14:38,982] INFO : Plotted number of elements +rank0 [2026-11-15 03:14:40,220] INFO : Plotted jets +rank0 [2026-11-15 03:14:41,570] INFO : Plotted jet ratio +rank0 [2026-11-15 03:15:39,218] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 03:15:39,218] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:16:28,758] INFO : Step 28000/500000 rank0 | Train Loss: 1.1354 | Train Loss Unweighted: 0.1801 | Train Loss Components: Classification: 0.0022 | Classification_binary: 0.1342 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0215 | Regression_eta: 0.0003 | Regression_pt: 0.0215 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.1630 | Classification_binary: 0.1342 | Regression_cos_phi: 0.0560 | Regression_energy: 0.1709 | Regression_eta: 0.0945 | Regression_pt: 0.1799 | Regression_sin_phi: 0.3369 | LR: 6.01e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4798s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0025 | max_backward: 0.0382 | max_optimizer_step: 0.0602 | max_tensorboard_logging: 0.1213 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3379.5 | uss_MB: 2151.3 | child_rss_MB: 22745.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:16:28,760] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 03:16:28,825] INFO : Running validation on rank0 +rank0 [2026-11-15 03:16:28,825] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:16:28,825] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:16:28,943] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 03:16:29,050] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:16:29,151] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:16:29,219] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:16:29,284] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:16:29,348] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:16:29,414] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:16:29,480] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:16:29,590] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:16:29,691] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:16:29,791] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:16:29,859] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:16:29,928] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:16:29,993] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:16:30,062] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:16:30,129] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:16:30,235] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:16:30,241] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:16:30,242] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:16:30,246] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:16:30,247] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:16:30,248] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:16:30,250] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:16:30,252] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:16:30,253] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:16:30,255] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:16:30,258] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:16:30,260] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:16:30,261] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:16:30,262] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:16:30,265] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:16:30,267] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:16:31,666] INFO : VALIDATION | Step=28000/500000 | Train Loss=1.1354 | Valid Loss=7.7273 | Train Loss Unweighted=0.1801 | Valid Loss Unweighted=2.0677 | Stale=17 | ETA=922.3m +rank0 [2026-11-15 03:16:31,666] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:16:31,737] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:16:31,744] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:16:31,791] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:16:32,254] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:16:32,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:16:32,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:16:32,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:16:33,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:16:33,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:16:33,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:16:33,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:16:34,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:16:34,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:16:34,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:16:34,895] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:16:35,139] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:16:35,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:16:35,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:16:35,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:16:35,891] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:16:35,896] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:16:36,426] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_28000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:16:36,779] INFO : Plotted number of elements +rank0 [2026-11-15 03:16:38,009] INFO : Plotted jets +rank0 [2026-11-15 03:16:39,357] INFO : Plotted jet ratio +rank0 [2026-11-15 03:16:50,336] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 03:16:50,336] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:17:50,216] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 03:17:50,217] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:18:26,690] INFO : Step 29000/500000 rank0 | Train Loss: 0.8729 | Train Loss Unweighted: 0.3614 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.3229 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0195 | Regression_eta: 0.0001 | Regression_pt: 0.0186 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0018 | Classification_binary: 0.3229 | Regression_cos_phi: 0.1202 | Regression_energy: 0.1548 | Regression_eta: 0.0469 | Regression_pt: 0.1553 | Regression_sin_phi: 0.0710 | LR: 6.29e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4845s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0357 | max_loss: 0.0026 | max_backward: 0.0437 | max_optimizer_step: 0.0631 | max_tensorboard_logging: 0.1240 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3405.8 | uss_MB: 2177.6 | child_rss_MB: 22745.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:18:26,692] INFO : Running validation on rank0 +rank0 [2026-11-15 03:18:26,692] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:18:26,692] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:18:26,809] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 03:18:26,912] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:18:27,013] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:18:27,083] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:18:27,149] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:18:27,214] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:18:27,279] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:18:27,345] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:18:27,456] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:18:27,558] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:18:27,658] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:18:27,726] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:18:27,792] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:18:27,857] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:18:27,924] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:18:27,991] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:18:28,098] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:18:28,103] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:18:28,107] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:18:28,109] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:18:28,110] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:18:28,113] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:18:28,114] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:18:28,116] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:18:28,118] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:18:28,120] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:18:28,122] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:18:28,124] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:18:28,125] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:18:28,126] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:18:28,129] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:18:28,130] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:18:29,530] INFO : VALIDATION | Step=29000/500000 | Train Loss=0.8729 | Valid Loss=7.8242 | Train Loss Unweighted=0.3614 | Valid Loss Unweighted=2.2341 | Stale=18 | ETA=920.5m +rank0 [2026-11-15 03:18:29,530] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:18:29,602] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:18:29,609] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:18:29,656] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:18:30,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:18:30,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:18:30,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:18:30,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:18:31,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:18:31,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:18:31,565] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:18:31,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:18:32,031] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:18:32,267] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:18:32,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:18:32,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:18:32,998] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:18:33,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:18:33,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:18:33,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:18:33,757] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:18:33,762] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:18:34,292] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_29000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:18:34,646] INFO : Plotted number of elements +rank0 [2026-11-15 03:18:35,877] INFO : Plotted jets +rank0 [2026-11-15 03:18:37,228] INFO : Plotted jet ratio +rank0 [2026-11-15 03:19:01,500] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 03:19:01,500] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:20:01,081] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 03:20:01,081] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:20:24,328] INFO : Step 30000/500000 rank0 | Train Loss: 0.8029 | Train Loss Unweighted: 0.1809 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.1504 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0150 | Regression_eta: 0.0002 | Regression_pt: 0.0148 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0115 | Classification_binary: 0.1504 | Regression_cos_phi: 0.1137 | Regression_energy: 0.1188 | Regression_eta: 0.0676 | Regression_pt: 0.1239 | Regression_sin_phi: 0.2170 | LR: 6.58e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4725s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0396 | max_optimizer_step: 0.0625 | max_tensorboard_logging: 0.1255 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3429.4 | uss_MB: 2201.3 | child_rss_MB: 22746.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:20:24,329] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 03:20:24,394] INFO : Running validation on rank0 +rank0 [2026-11-15 03:20:24,394] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:20:24,394] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:20:24,514] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 03:20:24,619] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:20:24,721] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:20:24,788] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:20:24,852] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:20:24,915] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:20:24,980] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:20:25,046] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:20:25,156] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:20:25,256] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:20:25,354] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:20:25,419] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:20:25,483] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:20:25,547] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:20:25,611] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:20:25,675] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:20:25,780] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:20:25,787] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:20:25,789] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:20:25,791] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:20:25,792] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:20:25,795] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:20:25,796] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:20:25,796] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:20:25,798] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:20:25,801] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:20:25,804] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:20:25,806] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:20:25,806] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:20:25,808] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:20:25,809] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:20:25,813] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:20:27,200] INFO : VALIDATION | Step=30000/500000 | Train Loss=0.8029 | Valid Loss=7.7593 | Train Loss Unweighted=0.1809 | Valid Loss Unweighted=2.1595 | Stale=19 | ETA=918.7m +rank0 [2026-11-15 03:20:27,201] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:20:27,275] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:20:27,283] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:27,330] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:20:27,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:20:28,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:20:28,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:20:28,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:20:28,756] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:20:28,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:20:29,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:20:29,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:20:29,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:20:29,953] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:20:30,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:20:30,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:20:30,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:20:30,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:20:31,176] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:20:31,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:20:31,455] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:20:31,461] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:31,986] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:32,340] INFO : Plotted number of elements +rank0 [2026-11-15 03:20:33,592] INFO : Plotted jets +rank0 [2026-11-15 03:20:34,940] INFO : Plotted jet ratio +rank0 [2026-11-15 03:21:12,447] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 03:21:12,447] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:22:12,413] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 03:22:12,413] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:22:22,388] INFO : Step 31000/500000 rank0 | Train Loss: 0.8978 | Train Loss Unweighted: 0.1203 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0605 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0288 | Regression_eta: 0.0002 | Regression_pt: 0.0305 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.0605 | Regression_cos_phi: 0.1069 | Regression_energy: 0.2286 | Regression_eta: 0.0608 | Regression_pt: 0.2546 | Regression_sin_phi: 0.1863 | LR: 6.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4884s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0405 | max_optimizer_step: 0.0622 | max_tensorboard_logging: 0.1329 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3380.7 | uss_MB: 2152.5 | child_rss_MB: 22746.2 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:22:22,390] INFO : Running validation on rank0 +rank0 [2026-11-15 03:22:22,390] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:22:22,390] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:22:22,507] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 03:22:22,614] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:22:22,714] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:22:22,782] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:22:22,848] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:22:22,912] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:22:22,977] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:22:23,044] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:22:23,157] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:22:23,258] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:22:23,358] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:22:23,426] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:22:23,492] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:22:23,558] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:22:23,625] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:22:23,692] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:22:23,800] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:22:23,807] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:22:23,808] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:22:23,811] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:22:23,812] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:22:23,815] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:22:23,816] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:22:23,818] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:22:23,820] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:22:23,822] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:22:23,822] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:22:23,826] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:22:23,826] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:22:23,828] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:22:23,830] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:22:23,832] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:22:25,236] INFO : VALIDATION | Step=31000/500000 | Train Loss=0.8978 | Valid Loss=8.0055 | Train Loss Unweighted=0.1203 | Valid Loss Unweighted=2.2648 | Stale=20 | ETA=916.9m +rank0 [2026-11-15 03:22:25,236] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:22:25,310] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:22:25,316] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:25,364] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:22:25,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:22:26,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:22:26,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:22:26,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:22:26,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:22:27,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:22:27,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:22:27,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:22:27,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:22:27,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:22:28,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:22:28,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:22:28,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:22:28,946] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:22:29,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:22:29,405] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:22:29,462] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:22:29,468] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:29,998] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_31000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:30,352] INFO : Plotted number of elements +rank0 [2026-11-15 03:22:31,591] INFO : Plotted jets +rank0 [2026-11-15 03:22:32,945] INFO : Plotted jet ratio +rank0 [2026-11-15 03:23:23,564] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-15 03:23:23,564] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:24:19,848] INFO : Step 32000/500000 rank0 | Train Loss: 1.0240 | Train Loss Unweighted: 0.1591 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.1016 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0286 | Regression_eta: 0.0002 | Regression_pt: 0.0283 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0058 | Classification_binary: 0.1016 | Regression_cos_phi: 0.1554 | Regression_energy: 0.2272 | Regression_eta: 0.0589 | Regression_pt: 0.2365 | Regression_sin_phi: 0.2387 | LR: 7.19e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4729s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0024 | max_backward: 0.0382 | max_optimizer_step: 0.0611 | max_tensorboard_logging: 0.1265 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3485.3 | uss_MB: 2257.1 | child_rss_MB: 22745.0 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:24:19,850] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-32000.pth +rank0 [2026-11-15 03:24:19,916] INFO : Running validation on rank0 +rank0 [2026-11-15 03:24:19,916] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:24:19,916] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:24:20,045] INFO : saved batch0_epoch32000.parquet +rank0 [2026-11-15 03:24:20,152] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:24:20,255] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:24:20,323] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:24:20,387] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:24:20,451] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:24:20,516] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:24:20,581] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:24:20,692] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:24:20,793] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:24:20,891] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:24:20,958] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:24:21,023] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:24:21,088] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:24:21,153] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:24:21,217] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:24:21,326] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:24:21,331] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:24:21,333] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:24:21,337] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:24:21,339] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:24:21,339] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:24:21,341] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:24:21,343] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:24:21,345] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:24:21,347] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:24:21,349] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:24:21,351] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:24:21,352] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:24:21,352] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:24:21,356] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:24:21,358] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:24:22,756] INFO : VALIDATION | Step=32000/500000 | Train Loss=1.0240 | Valid Loss=7.7241 | Train Loss Unweighted=0.1591 | Valid Loss Unweighted=2.3155 | Stale=21 | ETA=915.0m +rank0 [2026-11-15 03:24:22,757] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:24:22,830] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:24:22,838] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:22,886] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:24:23,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:24:23,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:24:23,865] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:24:24,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:24:24,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:24:24,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:24:24,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:24:25,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:24:25,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:24:25,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:24:25,785] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:24:26,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:24:26,277] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:24:26,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:24:26,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:24:26,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:24:27,032] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:24:27,038] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:27,565] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_32000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:27,920] INFO : Plotted number of elements +rank0 [2026-11-15 03:24:29,161] INFO : Plotted jets +rank0 [2026-11-15 03:24:30,549] INFO : Plotted jet ratio +rank0 [2026-11-15 03:24:34,508] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-15 03:24:34,508] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:25:34,402] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-15 03:25:34,402] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:26:17,781] INFO : Step 33000/500000 rank0 | Train Loss: 0.9192 | Train Loss Unweighted: 0.0681 | Train Loss Components: Classification: 0.0018 | Classification_binary: 0.0238 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0209 | Regression_eta: 0.0003 | Regression_pt: 0.0210 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.1336 | Classification_binary: 0.0238 | Regression_cos_phi: 0.1158 | Regression_energy: 0.1661 | Regression_eta: 0.1272 | Regression_pt: 0.1753 | Regression_sin_phi: 0.1774 | LR: 7.51e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4791s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0608 | max_tensorboard_logging: 0.1291 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3509.0 | uss_MB: 2280.9 | child_rss_MB: 22744.0 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:26:17,783] INFO : Running validation on rank0 +rank0 [2026-11-15 03:26:17,783] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:26:17,783] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:26:17,906] INFO : saved batch0_epoch33000.parquet +rank0 [2026-11-15 03:26:18,011] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:26:18,114] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:26:18,183] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:26:18,248] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:26:18,313] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:26:18,379] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:26:18,445] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:26:18,558] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:26:18,659] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:26:18,759] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:26:18,825] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:26:18,891] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:26:18,955] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:26:19,020] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:26:19,088] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:26:19,199] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:26:19,205] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:26:19,208] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:26:19,210] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:26:19,212] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:26:19,212] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:26:19,214] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:26:19,215] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:26:19,218] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:26:19,220] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:26:19,222] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:26:19,224] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:26:19,225] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:26:19,225] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:26:19,227] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:26:19,231] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:26:20,631] INFO : VALIDATION | Step=33000/500000 | Train Loss=0.9192 | Valid Loss=7.6376 | Train Loss Unweighted=0.0681 | Valid Loss Unweighted=2.4198 | Stale=22 | ETA=913.2m +rank0 [2026-11-15 03:26:20,631] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:26:20,705] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:26:20,711] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:20,760] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:26:21,246] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:26:21,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:26:21,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:26:21,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:26:22,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:26:22,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:26:22,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:26:22,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:26:23,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:26:23,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:26:23,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:26:23,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:26:24,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:26:24,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:26:24,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:26:24,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:26:24,894] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:26:24,900] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:25,432] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_33000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:25,782] INFO : Plotted number of elements +rank0 [2026-11-15 03:26:27,027] INFO : Plotted jets +rank0 [2026-11-15 03:26:28,387] INFO : Plotted jet ratio +rank0 [2026-11-15 03:26:45,505] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-15 03:26:45,505] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:27:45,172] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-15 03:27:45,173] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:28:15,238] INFO : Step 34000/500000 rank0 | Train Loss: 0.5505 | Train Loss Unweighted: 0.0645 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.0439 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0101 | Regression_eta: 0.0001 | Regression_pt: 0.0101 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0052 | Classification_binary: 0.0439 | Regression_cos_phi: 0.1097 | Regression_energy: 0.0802 | Regression_eta: 0.0442 | Regression_pt: 0.0840 | Regression_sin_phi: 0.1834 | LR: 7.83e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4810s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0024 | max_backward: 0.0383 | max_optimizer_step: 0.0607 | max_tensorboard_logging: 0.1324 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3453.7 | uss_MB: 2225.6 | child_rss_MB: 22740.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:28:15,240] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-34000.pth +rank0 [2026-11-15 03:28:15,306] INFO : Running validation on rank0 +rank0 [2026-11-15 03:28:15,306] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:28:15,306] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:28:15,432] INFO : saved batch0_epoch34000.parquet +rank0 [2026-11-15 03:28:15,540] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:28:15,640] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:28:15,709] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:28:15,775] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:28:15,839] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:28:15,905] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:28:15,971] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:28:16,086] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:28:16,188] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:28:16,287] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:28:16,354] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:28:16,419] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:28:16,485] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:28:16,551] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:28:16,617] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:28:16,728] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:28:16,735] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:28:16,737] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:28:16,739] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:28:16,741] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:28:16,743] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:28:16,744] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:28:16,746] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:28:16,746] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:28:16,748] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:28:16,750] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:28:16,752] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:28:16,754] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:28:16,756] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:28:16,758] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:28:16,759] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:28:18,167] INFO : VALIDATION | Step=34000/500000 | Train Loss=0.5505 | Valid Loss=8.3962 | Train Loss Unweighted=0.0645 | Valid Loss Unweighted=2.3623 | Stale=23 | ETA=911.3m +rank0 [2026-11-15 03:28:18,168] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:28:18,241] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:28:18,249] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:18,296] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:28:18,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:28:19,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:28:19,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:28:19,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:28:19,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:28:19,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:28:20,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:28:20,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:28:20,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:28:20,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:28:21,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:28:21,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:28:21,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:28:21,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:28:22,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:28:22,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:28:22,418] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:28:22,424] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:22,950] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_34000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:23,307] INFO : Plotted number of elements +rank0 [2026-11-15 03:28:24,536] INFO : Plotted jets +rank0 [2026-11-15 03:28:25,903] INFO : Plotted jet ratio +rank0 [2026-11-15 03:28:56,308] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-15 03:28:56,308] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:29:56,260] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-15 03:29:56,260] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:30:13,333] INFO : Step 35000/500000 rank0 | Train Loss: 1.6985 | Train Loss Unweighted: 0.4161 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.3627 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0252 | Regression_eta: 0.0013 | Regression_pt: 0.0263 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0165 | Classification_binary: 0.3627 | Regression_cos_phi: 0.1768 | Regression_energy: 0.1998 | Regression_eta: 0.4612 | Regression_pt: 0.2199 | Regression_sin_phi: 0.2616 | LR: 8.16e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4809s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0025 | max_backward: 0.0396 | max_optimizer_step: 0.0640 | max_tensorboard_logging: 0.1243 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3558.1 | uss_MB: 2329.9 | child_rss_MB: 22747.0 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:30:13,334] INFO : Running validation on rank0 +rank0 [2026-11-15 03:30:13,334] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:30:13,334] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:30:13,453] INFO : saved batch0_epoch35000.parquet +rank0 [2026-11-15 03:30:13,556] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:30:13,655] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:30:13,723] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:30:13,788] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:30:13,853] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:30:13,918] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:30:13,985] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:30:14,098] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:30:14,200] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:30:14,300] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:30:14,368] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:30:14,433] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:30:14,499] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:30:14,566] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:30:14,634] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:30:14,745] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:30:14,751] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:30:14,754] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:30:14,755] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:30:14,758] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:30:14,760] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:30:14,761] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:30:14,763] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:30:14,764] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:30:14,765] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:30:14,767] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:30:14,770] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:30:14,771] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:30:14,773] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:30:14,775] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:30:14,776] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:30:16,183] INFO : VALIDATION | Step=35000/500000 | Train Loss=1.6985 | Valid Loss=8.4200 | Train Loss Unweighted=0.4161 | Valid Loss Unweighted=2.5928 | Stale=24 | ETA=909.5m +rank0 [2026-11-15 03:30:16,183] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:30:16,255] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:30:16,261] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:16,309] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:30:16,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:30:17,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:30:17,254] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:30:17,496] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:30:17,737] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:30:17,979] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:30:18,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:30:18,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:30:18,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:30:18,924] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:30:19,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:30:19,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:30:19,654] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:30:19,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:30:20,133] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:30:20,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:30:20,409] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:30:20,415] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:20,948] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_35000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:21,307] INFO : Plotted number of elements +rank0 [2026-11-15 03:30:22,539] INFO : Plotted jets +rank0 [2026-11-15 03:30:23,905] INFO : Plotted jet ratio +rank0 [2026-11-15 03:31:07,580] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-15 03:31:07,580] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:32:07,428] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-15 03:32:07,428] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:32:11,302] INFO : Step 36000/500000 rank0 | Train Loss: 0.8054 | Train Loss Unweighted: 0.3554 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.3186 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0179 | Regression_eta: 0.0002 | Regression_pt: 0.0187 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0000 | Classification_binary: 0.3186 | Regression_cos_phi: 0.0416 | Regression_energy: 0.1418 | Regression_eta: 0.0733 | Regression_pt: 0.1559 | Regression_sin_phi: 0.0742 | LR: 8.50e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5724s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0344 | max_loss: 0.0024 | max_backward: 0.0380 | max_optimizer_step: 0.0598 | max_tensorboard_logging: 0.2263 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3581.0 | uss_MB: 2360.5 | child_rss_MB: 22746.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:32:11,305] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-36000.pth +rank0 [2026-11-15 03:32:11,376] INFO : Running validation on rank0 +rank0 [2026-11-15 03:32:11,376] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:32:11,376] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:32:11,497] INFO : saved batch0_epoch36000.parquet +rank0 [2026-11-15 03:32:11,600] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:32:11,699] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:32:11,766] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:32:11,831] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:32:11,894] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:32:11,959] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:32:12,025] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:32:12,139] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:32:12,239] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:32:12,341] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:32:12,407] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:32:12,472] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:32:12,538] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:32:12,605] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:32:12,672] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:32:12,786] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:32:12,793] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:32:12,795] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:32:12,797] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:32:12,799] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:32:12,801] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:32:12,802] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:32:12,804] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:32:12,805] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:32:12,808] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:32:12,809] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:32:12,810] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:32:12,812] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:32:12,814] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:32:12,816] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:32:12,820] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:32:14,219] INFO : VALIDATION | Step=36000/500000 | Train Loss=0.8054 | Valid Loss=8.4207 | Train Loss Unweighted=0.3554 | Valid Loss Unweighted=2.5628 | Stale=25 | ETA=907.6m +rank0 [2026-11-15 03:32:14,219] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:32:14,333] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:32:14,340] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:32:14,395] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:32:14,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:32:15,099] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:32:15,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:32:15,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:32:15,831] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:32:16,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:32:16,319] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:32:16,555] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:32:16,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:32:17,031] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:32:17,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:32:17,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:32:17,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:32:18,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:32:18,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:32:18,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:32:18,527] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:32:18,534] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:32:19,068] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_36000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:32:19,423] INFO : Plotted number of elements +rank0 [2026-11-15 03:32:20,658] INFO : Plotted jets +rank0 [2026-11-15 03:32:22,045] INFO : Plotted jet ratio +rank0 [2026-11-15 03:33:18,892] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-15 03:33:18,892] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:34:09,189] INFO : Step 37000/500000 rank0 | Train Loss: 1.0400 | Train Loss Unweighted: 0.2912 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.2189 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0357 | Regression_eta: 0.0002 | Regression_pt: 0.0362 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.2189 | Regression_cos_phi: 0.1056 | Regression_energy: 0.2834 | Regression_eta: 0.0662 | Regression_pt: 0.3026 | Regression_sin_phi: 0.0631 | LR: 8.85e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4882s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0359 | max_loss: 0.0027 | max_backward: 0.0387 | max_optimizer_step: 0.0651 | max_tensorboard_logging: 0.1195 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3602.1 | uss_MB: 2374.2 | child_rss_MB: 22745.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:34:09,190] INFO : Running validation on rank0 +rank0 [2026-11-15 03:34:09,191] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:34:09,191] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:34:09,310] INFO : saved batch0_epoch37000.parquet +rank0 [2026-11-15 03:34:09,415] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:34:09,517] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:34:09,584] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:34:09,649] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:34:09,714] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:34:09,781] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:34:09,849] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:34:09,963] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:34:10,066] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:34:10,196] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:34:10,264] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:34:10,330] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:34:10,398] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:34:10,463] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:34:10,527] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:34:10,637] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:34:10,643] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:34:10,644] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:34:10,646] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:34:10,648] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:34:10,650] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:34:10,652] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:34:10,654] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:34:10,656] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:34:10,658] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:34:10,660] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:34:10,662] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:34:10,662] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:34:10,664] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:34:10,666] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:34:10,668] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:34:12,063] INFO : VALIDATION | Step=37000/500000 | Train Loss=1.0400 | Valid Loss=8.3483 | Train Loss Unweighted=0.2912 | Valid Loss Unweighted=2.7325 | Stale=26 | ETA=905.8m +rank0 [2026-11-15 03:34:12,064] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:34:12,132] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:34:12,137] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:34:12,186] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:34:12,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:34:12,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:34:13,161] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:34:13,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:34:13,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:34:13,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:34:14,115] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:34:14,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:34:14,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:34:14,829] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:34:15,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:34:15,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:34:15,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:34:15,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:34:16,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:34:16,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:34:16,289] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:34:16,295] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:34:16,830] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_37000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:34:17,182] INFO : Plotted number of elements +rank0 [2026-11-15 03:34:18,402] INFO : Plotted jets +rank0 [2026-11-15 03:34:19,755] INFO : Plotted jet ratio +rank0 [2026-11-15 03:34:29,819] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-15 03:34:29,819] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:35:29,345] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-15 03:35:29,345] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:36:06,372] INFO : Step 38000/500000 rank0 | Train Loss: 0.9542 | Train Loss Unweighted: 0.0576 | Train Loss Components: Classification: 0.0008 | Classification_binary: 0.0257 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0152 | Regression_eta: 0.0002 | Regression_pt: 0.0151 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.0257 | Regression_cos_phi: 0.2645 | Regression_energy: 0.1211 | Regression_eta: 0.0791 | Regression_pt: 0.1258 | Regression_sin_phi: 0.2816 | LR: 9.21e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4670s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0026 | max_backward: 0.0394 | max_optimizer_step: 0.0617 | max_tensorboard_logging: 0.1153 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3630.8 | uss_MB: 2402.9 | child_rss_MB: 22745.1 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:36:06,374] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-38000.pth +rank0 [2026-11-15 03:36:06,442] INFO : Running validation on rank0 +rank0 [2026-11-15 03:36:06,442] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:36:06,442] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:36:06,561] INFO : saved batch0_epoch38000.parquet +rank0 [2026-11-15 03:36:06,664] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:36:06,767] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:36:06,833] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:36:06,897] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:36:06,960] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:36:07,026] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:36:07,093] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:36:07,204] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:36:07,304] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:36:07,402] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:36:07,468] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:36:07,532] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:36:07,596] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:36:07,660] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:36:07,724] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:36:07,832] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:36:07,839] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:36:07,839] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:36:07,843] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:36:07,844] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:36:07,847] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:36:07,847] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:36:07,848] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:36:07,851] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:36:07,852] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:36:07,855] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:36:07,857] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:36:07,858] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:36:07,858] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:36:07,860] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:36:07,864] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:36:09,251] INFO : VALIDATION | Step=38000/500000 | Train Loss=0.9542 | Valid Loss=9.3149 | Train Loss Unweighted=0.0576 | Valid Loss Unweighted=2.8578 | Stale=27 | ETA=903.8m +rank0 [2026-11-15 03:36:09,252] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:36:09,321] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:36:09,324] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:36:09,372] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:36:09,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:36:10,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:36:10,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:36:10,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:36:10,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:36:11,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:36:11,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:36:11,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:36:11,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:36:11,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:36:12,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:36:12,475] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:36:12,716] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:36:12,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:36:13,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:36:13,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:36:13,476] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:36:13,482] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:36:14,011] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_38000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:36:14,365] INFO : Plotted number of elements +rank0 [2026-11-15 03:36:15,596] INFO : Plotted jets +rank0 [2026-11-15 03:36:16,961] INFO : Plotted jet ratio +rank0 [2026-11-15 03:36:40,353] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-15 03:36:40,353] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:37:39,953] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-15 03:37:39,953] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:38:04,058] INFO : Step 39000/500000 rank0 | Train Loss: 0.6835 | Train Loss Unweighted: 0.1301 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0964 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0168 | Regression_eta: 0.0002 | Regression_pt: 0.0164 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0000 | Classification_binary: 0.0964 | Regression_cos_phi: 0.1282 | Regression_energy: 0.1336 | Regression_eta: 0.0554 | Regression_pt: 0.1373 | Regression_sin_phi: 0.1326 | LR: 9.57e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4995s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0026 | max_backward: 0.0395 | max_optimizer_step: 0.0628 | max_tensorboard_logging: 0.1231 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3654.1 | uss_MB: 2426.2 | child_rss_MB: 22746.9 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:38:04,060] INFO : Running validation on rank0 +rank0 [2026-11-15 03:38:04,060] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:38:04,060] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:38:04,177] INFO : saved batch0_epoch39000.parquet +rank0 [2026-11-15 03:38:04,283] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:38:04,383] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:38:04,452] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:38:04,517] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:38:04,580] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:38:04,645] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:38:04,710] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:38:04,822] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:38:04,925] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:38:05,025] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:38:05,094] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:38:05,160] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:38:05,225] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:38:05,292] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:38:05,359] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:38:05,472] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:38:05,479] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:38:05,479] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:38:05,483] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:38:05,484] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:38:05,487] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:38:05,488] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:38:05,490] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:38:05,492] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:38:05,494] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:38:05,496] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:38:05,498] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:38:05,499] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:38:05,501] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:38:05,501] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:38:05,503] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:38:06,893] INFO : VALIDATION | Step=39000/500000 | Train Loss=0.6835 | Valid Loss=8.2446 | Train Loss Unweighted=0.1301 | Valid Loss Unweighted=2.7478 | Stale=28 | ETA=901.9m +rank0 [2026-11-15 03:38:06,894] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:38:06,965] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:38:06,968] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:38:07,017] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:38:07,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:38:07,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:38:07,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:38:08,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:38:08,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:38:08,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:38:08,946] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:38:09,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:38:09,417] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:38:09,654] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:38:09,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:38:10,140] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:38:10,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:38:10,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:38:10,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:38:11,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:38:11,143] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:38:11,148] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:38:11,675] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_39000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:38:12,029] INFO : Plotted number of elements +rank0 [2026-11-15 03:38:13,273] INFO : Plotted jets +rank0 [2026-11-15 03:38:14,651] INFO : Plotted jet ratio +rank0 [2026-11-15 03:38:51,237] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-15 03:38:51,237] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:39:50,979] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-15 03:39:50,979] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:40:01,787] INFO : Step 40000/500000 rank0 | Train Loss: 1.0405 | Train Loss Unweighted: 0.0667 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0161 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0252 | Regression_eta: 0.0002 | Regression_pt: 0.0246 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0002 | Classification_binary: 0.0161 | Regression_cos_phi: 0.3844 | Regression_energy: 0.2000 | Regression_eta: 0.0760 | Regression_pt: 0.2056 | Regression_sin_phi: 0.1581 | LR: 9.94e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4705s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0617 | max_tensorboard_logging: 0.1205 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3686.5 | uss_MB: 2458.7 | child_rss_MB: 22745.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:40:01,789] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/checkpoints/checkpoint-40000.pth +rank0 [2026-11-15 03:40:01,856] INFO : Running validation on rank0 +rank0 [2026-11-15 03:40:01,856] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:40:01,856] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:40:01,974] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-15 03:40:02,078] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:40:02,182] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:40:02,250] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:40:02,317] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:40:02,383] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:40:02,451] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:40:02,521] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:40:02,636] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:40:02,737] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:40:02,837] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:40:02,903] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:40:02,969] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:40:03,034] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:40:03,101] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:40:03,165] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:40:03,273] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:40:03,278] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:40:03,282] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:40:03,283] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:40:03,284] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:40:03,286] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:40:03,288] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:40:03,290] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:40:03,292] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:40:03,294] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:40:03,296] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:40:03,298] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:40:03,299] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:40:03,301] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:40:03,302] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:40:03,305] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:40:04,697] INFO : VALIDATION | Step=40000/500000 | Train Loss=1.0405 | Valid Loss=8.6351 | Train Loss Unweighted=0.0667 | Valid Loss Unweighted=2.7612 | Stale=29 | ETA=900.0m +rank0 [2026-11-15 03:40:04,697] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:40:04,768] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:40:04,772] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:40:04,823] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:40:05,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:40:05,607] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:40:05,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:40:06,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:40:06,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:40:06,554] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:40:06,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:40:07,047] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:40:07,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:40:07,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:40:07,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:40:08,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:40:08,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:40:08,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:40:08,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:40:08,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:40:08,991] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:40:08,996] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:40:09,526] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:40:09,866] INFO : Plotted number of elements +rank0 [2026-11-15 03:40:11,105] INFO : Plotted jets +rank0 [2026-11-15 03:40:12,458] INFO : Plotted jet ratio +rank0 [2026-11-15 03:41:02,112] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-15 03:41:02,112] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:41:59,390] INFO : Step 41000/500000 rank0 | Train Loss: 1.0021 | Train Loss Unweighted: 0.1107 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.0625 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0238 | Regression_eta: 0.0003 | Regression_pt: 0.0234 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 74.7935 | Classification_binary: 1.0000 | Regression_cos_phi: 800.2740 | Regression_energy: 7.9446 | Regression_eta: 368.0083 | Regression_pt: 8.3527 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0122 | Classification_binary: 0.0625 | Regression_cos_phi: 0.2292 | Regression_energy: 0.1894 | Regression_eta: 0.1114 | Regression_pt: 0.1952 | Regression_sin_phi: 0.2022 | LR: 1.03e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.4810s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0354 | max_loss: 0.0026 | max_backward: 0.0390 | max_optimizer_step: 0.0622 | max_tensorboard_logging: 0.1244 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3695.8 | uss_MB: 2468.0 | child_rss_MB: 22745.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:41:59,392] INFO : Running validation on rank0 +rank0 [2026-11-15 03:41:59,392] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:41:59,392] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:41:59,513] INFO : saved batch0_epoch41000.parquet +rank0 [2026-11-15 03:41:59,618] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:41:59,717] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:41:59,785] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:41:59,850] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:41:59,915] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:41:59,980] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:42:00,047] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:42:00,161] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:42:00,261] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:42:00,360] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:42:00,426] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:42:00,492] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:42:00,555] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:42:00,621] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:42:00,686] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:42:00,797] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:42:00,802] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:42:00,804] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:42:00,807] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:42:00,809] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:42:00,811] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:42:00,812] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:42:00,814] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:42:00,814] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:42:00,816] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:42:00,818] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:42:00,820] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:42:00,822] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:42:00,824] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:42:00,826] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:42:00,828] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:42:02,240] INFO : VALIDATION | Step=41000/500000 | Train Loss=1.0021 | Valid Loss=8.3927 | Train Loss Unweighted=0.1107 | Valid Loss Unweighted=2.7661 | Stale=30 | ETA=898.1m +rank0 [2026-11-15 03:42:02,241] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:42:02,313] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:42:02,316] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:42:02,365] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:42:02,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:42:03,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:42:03,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:42:03,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:42:03,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:42:04,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:42:04,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:42:04,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:42:04,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:42:04,991] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:42:05,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:42:05,474] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:42:05,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:42:05,952] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:42:06,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:42:06,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:42:06,469] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:42:06,485] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:42:07,012] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2mm_seed12345_20261115_022113_469398/preds_step_41000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:42:07,366] INFO : Plotted number of elements +rank0 [2026-11-15 03:42:08,613] INFO : Plotted jets +rank0 [2026-11-15 03:42:09,982] INFO : Plotted jet ratio +rank0 [2026-11-15 03:42:10,319] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 03:42:10,320] INFO : Training completed. Total time on device 0: 80.356min diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..19389f275daa1d9e284f8439e7d8d682a02a21f4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6738b5dab46ba6e7b515b14aedc373ef58fe732c15c5def16a14d69206de7912 +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-34000.pth b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-34000.pth new file mode 100644 index 0000000000000000000000000000000000000000..3eab7999b508b4f2861b685d416e26ad93bf2ac9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-34000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e91fa54563e442e31306c4240c9fc193f54434b7a99e410a8562a4fab3ef189d +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_1000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_1000.json new file mode 100644 index 0000000000000000000000000000000000000000..3b9699e3b795ccf020f9b856c420f4f45e5b6cf5 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_1000.json @@ -0,0 +1 @@ +{"epoch": 1.7793594306049823, "train": {"Classification_binary": 1.0054813623428345, "Classification": 0.007555863820016384, "Regression_pt": 0.04992888122797012, "Regression_eta": 0.003269694745540619, "Regression_sin_phi": 0.0008850073791109025, "Regression_cos_phi": 0.0018451070645824075, "Regression_energy": 0.0377088226377964, "Total": 5.3462419509887695, "Total_unweighted": 1.1066747392178513}, "train_group": {}, "valid": {"Classification_binary": 1.1285297870635986, "Classification": 0.009873494505882263, "Regression_pt": 0.10161291062831879, "Regression_eta": 0.0011232863180339336, "Regression_sin_phi": 0.0042585101909935474, "Regression_cos_phi": 0.0018296574708074331, "Regression_energy": 0.10192592442035675, "Total": 8.895478248596191, "Total_unweighted": 1.3491535705979913, "metrics/particle/class_1/efficiency": 0.970251739025116, "metrics/particle/class_1/pid_accuracy": 0.9834905862808228, "metrics/particle/class_2/efficiency": 0.5185185074806213, "metrics/particle/class_2/pid_accuracy": 0.8571428656578064, "metrics/particle/class_3/efficiency": 0.8464646339416504, "metrics/particle/class_3/pid_accuracy": 0.9976133704185486, "metrics/particle/class_4/efficiency": 0.9322034120559692, "metrics/particle/class_4/pid_accuracy": 0.9272727370262146, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 0.988095223903656, "metrics/particle/count/bias_mean": -0.02800000086426735, "metrics/particle/count/mae": 0.2680000066757202, "metrics/particle/count/prediction_mean": 4.619999885559082, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.049791544675827026, "metrics/particle/event/energy_response_mean": 0.9984227418899536, "metrics/particle/event/met_abs_error": 1.1654754877090454, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04856681823730469, "metrics/particle/event/scalar_pt_response_mean": 0.9982523918151855, "metrics/particle/event/vector_pt_closure": 0.045495375990867615, "metrics/particle/matched/delta_eta_abs_mean": 0.007745427079498768, "metrics/particle/matched/delta_phi_abs_mean": 0.007021756377071142, "metrics/particle/matched/delta_r_mean": 0.011648687534034252, "metrics/particle/matched/energy_relative_abs_mean": 0.06116979941725731, "metrics/particle/matched/pid_accuracy": 0.9819219708442688, "metrics/particle/matched/pt_relative_abs_mean": 0.06133931502699852, "metrics/particle/matching/duplicate_fraction": 0.0181818176060915, "metrics/particle/matching/efficiency": 0.9044750332832336, "metrics/particle/matching/f1": 0.9072076082229614, "metrics/particle/matching/purity": 0.909956693649292, "metrics/particle/matching/target_coverage_dr0p05": 0.8967297673225403, "metrics/particle/matching/target_coverage_dr0p10": 0.9388984441757202, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8407917618751526, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9225473403930664}, "step_train_time": 0.45124316215515137, "step_valid_time": 2.617801464999957, "step_total_time": 111.11541438102722, "elapsed_training_loop_seconds": 111.11541438102722, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0038672905142887, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.030361585197947072, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9895470383275261, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.009302325581395349, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.986046511627907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9849012775842044, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.470900107000034, "step_plot_time": 4.064287262999869, "step_validation_cycle_time": 11.58254643400005} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..169dfffbda888e2ea5332527dae44b7c8068bf28 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 0.4421354830265045, "Classification": 0.0007503919769078493, "Regression_pt": 0.020399220287799835, "Regression_eta": 0.00012129917740821838, "Regression_sin_phi": 9.639574273023754e-05, "Regression_cos_phi": 0.00014568216283805668, "Regression_energy": 0.020556695759296417, "Total": 1.1074926853179932, "Total_unweighted": 0.48420516813348513}, "train_group": {}, "valid": {"Classification_binary": 1.1111950874328613, "Classification": 0.009974075481295586, "Regression_pt": 0.09495466947555542, "Regression_eta": 0.0009128209203481674, "Regression_sin_phi": 0.0036079457495361567, "Regression_cos_phi": 0.0018098939908668399, "Regression_energy": 0.09435528516769409, "Total": 8.165559768676758, "Total_unweighted": 1.3168097782181576, "metrics/particle/class_1/efficiency": 0.9794050455093384, "metrics/particle/class_1/pid_accuracy": 0.9766355156898499, "metrics/particle/class_2/efficiency": 0.5925925970077515, "metrics/particle/class_2/pid_accuracy": 0.875, "metrics/particle/class_3/efficiency": 0.8767676949501038, "metrics/particle/class_3/pid_accuracy": 0.9907833933830261, "metrics/particle/class_4/efficiency": 0.9406779408454895, "metrics/particle/class_4/pid_accuracy": 0.9459459185600281, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.012000000104308128, "metrics/particle/count/mae": 0.21199999749660492, "metrics/particle/count/prediction_mean": 4.636000156402588, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04229109361767769, "metrics/particle/event/energy_response_mean": 1.000394582748413, "metrics/particle/event/met_abs_error": 1.0272409915924072, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04108031094074249, "metrics/particle/event/scalar_pt_response_mean": 0.9988605380058289, "metrics/particle/event/vector_pt_closure": 0.04191042110323906, "metrics/particle/matched/delta_eta_abs_mean": 0.007293302565813065, "metrics/particle/matched/delta_phi_abs_mean": 0.006459326948970556, "metrics/particle/matched/delta_r_mean": 0.010890673846006393, "metrics/particle/matched/energy_relative_abs_mean": 0.06276989728212357, "metrics/particle/matched/pid_accuracy": 0.9794967174530029, "metrics/particle/matched/pt_relative_abs_mean": 0.06353099644184113, "metrics/particle/matching/duplicate_fraction": 0.01294219121336937, "metrics/particle/matching/efficiency": 0.9234079122543335, "metrics/particle/matching/f1": 0.9246014356613159, "metrics/particle/matching/purity": 0.9257981181144714, "metrics/particle/matching/target_coverage_dr0p05": 0.9130808711051941, "metrics/particle/matching/target_coverage_dr0p10": 0.9466437101364136, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8476763963699341, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9406196475028992}, "step_train_time": 0.4579763412475586, "step_valid_time": 2.843260397999984, "step_total_time": 1172.932246208191, "elapsed_training_loop_seconds": 1172.932246208191, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001563828301282, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.03086459132632524, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 432, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9884259259259259, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9895712630359212, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.011574074074074073, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9861111111111112, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9872537659327926, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.153542276999815, "step_plot_time": 3.483996776999902, "step_validation_cycle_time": 10.796744471999773} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_11000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_11000.json new file mode 100644 index 0000000000000000000000000000000000000000..d1c57a50fd4bd86be6b481bda8876a671be6f968 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_11000.json @@ -0,0 +1 @@ +{"epoch": 19.572953736654803, "train": {"Classification_binary": 0.25348809361457825, "Classification": 0.00044928572606295347, "Regression_pt": 0.028642896562814713, "Regression_eta": 0.00011873246694449335, "Regression_sin_phi": 0.0001388772070640698, "Regression_cos_phi": 7.080564682837576e-05, "Regression_energy": 0.028846178203821182, "Total": 1.0295723676681519, "Total_unweighted": 0.31175486942811403}, "train_group": {}, "valid": {"Classification_binary": 1.1250057220458984, "Classification": 0.01070127822458744, "Regression_pt": 0.10063285380601883, "Regression_eta": 0.0010199438547715545, "Regression_sin_phi": 0.009004117920994759, "Regression_cos_phi": 0.004279745742678642, "Regression_energy": 0.10076846927404404, "Total": 14.473020553588867, "Total_unweighted": 1.3514121308689937, "metrics/particle/class_1/efficiency": 0.9771167039871216, "metrics/particle/class_1/pid_accuracy": 0.9836065769195557, "metrics/particle/class_2/efficiency": 0.5185185074806213, "metrics/particle/class_2/pid_accuracy": 0.9285714030265808, "metrics/particle/class_3/efficiency": 0.870707094669342, "metrics/particle/class_3/pid_accuracy": 0.9930394291877747, "metrics/particle/class_4/efficiency": 0.9406779408454895, "metrics/particle/class_4/pid_accuracy": 0.954954981803894, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": 0.0, "metrics/particle/count/mae": 0.2240000069141388, "metrics/particle/count/prediction_mean": 4.6479997634887695, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04222485050559044, "metrics/particle/event/energy_response_mean": 1.005284070968628, "metrics/particle/event/met_abs_error": 1.064369559288025, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04078349843621254, "metrics/particle/event/scalar_pt_response_mean": 1.0037604570388794, "metrics/particle/event/vector_pt_closure": 0.0408085398375988, "metrics/particle/matched/delta_eta_abs_mean": 0.0071395402774214745, "metrics/particle/matched/delta_phi_abs_mean": 0.006289201322942972, "metrics/particle/matched/delta_r_mean": 0.010558225214481354, "metrics/particle/matched/energy_relative_abs_mean": 0.0628301277756691, "metrics/particle/matched/pid_accuracy": 0.9850046634674072, "metrics/particle/matched/pt_relative_abs_mean": 0.0630144476890564, "metrics/particle/matching/duplicate_fraction": 0.017211703583598137, "metrics/particle/matching/efficiency": 0.9182444214820862, "metrics/particle/matching/f1": 0.9182444214820862, "metrics/particle/matching/purity": 0.9182444214820862, "metrics/particle/matching/target_coverage_dr0p05": 0.9079173803329468, "metrics/particle/matching/target_coverage_dr0p10": 0.9432013630867004, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8554216623306274, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9423407912254333}, "step_train_time": 0.4658961296081543, "step_valid_time": 2.8299664800001665, "step_total_time": 1290.7404296398163, "elapsed_training_loop_seconds": 1290.7404296398163, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9991893108071807, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.029246000479811296, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 432, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9814814814814815, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9826187717265353, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.018518518518518517, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9791666666666666, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9803012746234068, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9837587006960556, "step_test_time": 4.159352719000253, "step_plot_time": 3.5111541430001125, "step_validation_cycle_time": 10.813721091999923} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_12000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_12000.json new file mode 100644 index 0000000000000000000000000000000000000000..4ab2103ee6c6a8a23af6e295555808154c923478 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_12000.json @@ -0,0 +1 @@ +{"epoch": 21.352313167259787, "train": {"Classification_binary": 0.3255994915962219, "Classification": 0.0002611930249258876, "Regression_pt": 0.022583061829209328, "Regression_eta": 0.00019160307419952005, "Regression_sin_phi": 0.00026057905051857233, "Regression_cos_phi": 0.0006048025097697973, "Regression_energy": 0.021869247779250145, "Total": 1.465002179145813, "Total_unweighted": 0.3713699788640952}, "train_group": {}, "valid": {"Classification_binary": 1.1486659049987793, "Classification": 0.01184308622032404, "Regression_pt": 0.10037815570831299, "Regression_eta": 0.0010983289685100317, "Regression_sin_phi": 0.006241020746529102, "Regression_cos_phi": 0.003076933091506362, "Regression_energy": 0.10024893283843994, "Total": 11.53114128112793, "Total_unweighted": 1.3715523625724018, "metrics/particle/class_1/efficiency": 0.9794050455093384, "metrics/particle/class_1/pid_accuracy": 0.9813084006309509, "metrics/particle/class_2/efficiency": 0.4444444477558136, "metrics/particle/class_2/pid_accuracy": 1.0, "metrics/particle/class_3/efficiency": 0.8646464943885803, "metrics/particle/class_3/pid_accuracy": 0.9906542301177979, "metrics/particle/class_4/efficiency": 0.9491525292396545, "metrics/particle/class_4/pid_accuracy": 0.9464285969734192, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.024000000208616257, "metrics/particle/count/mae": 0.2160000056028366, "metrics/particle/count/prediction_mean": 4.624000072479248, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04363391920924187, "metrics/particle/event/energy_response_mean": 1.0003254413604736, "metrics/particle/event/met_abs_error": 1.075439691543579, "metrics/particle/event/scalar_pt_relative_abs_error": 0.042360831052064896, "metrics/particle/event/scalar_pt_response_mean": 0.9989809393882751, "metrics/particle/event/vector_pt_closure": 0.04232625663280487, "metrics/particle/matched/delta_eta_abs_mean": 0.0069787772372365, "metrics/particle/matched/delta_phi_abs_mean": 0.006933134980499744, "metrics/particle/matched/delta_r_mean": 0.010984339751303196, "metrics/particle/matched/energy_relative_abs_mean": 0.060784004628658295, "metrics/particle/matched/pid_accuracy": 0.9830827116966248, "metrics/particle/matched/pt_relative_abs_mean": 0.060921527445316315, "metrics/particle/matching/duplicate_fraction": 0.012110726907849312, "metrics/particle/matching/efficiency": 0.9156626462936401, "metrics/particle/matching/f1": 0.9180327653884888, "metrics/particle/matching/purity": 0.9204152226448059, "metrics/particle/matching/target_coverage_dr0p05": 0.9130808711051941, "metrics/particle/matching/target_coverage_dr0p10": 0.9423407912254333, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.846815824508667, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9388984441757202}, "step_train_time": 0.45340824127197266, "step_valid_time": 2.8141322680003213, "step_total_time": 1409.065720796585, "elapsed_training_loop_seconds": 1409.065720796585, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9982149905879609, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.031025459987783832, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9906759906759907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.009324009324009324, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.986013986013986, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9837209302325581, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9860788863109049, "step_test_time": 4.228578282000399, "step_plot_time": 3.535804434000056, "step_validation_cycle_time": 10.898242754999956} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_13000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_13000.json new file mode 100644 index 0000000000000000000000000000000000000000..9537bde627d313737222418fc5943e2a54f2f704 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_13000.json @@ -0,0 +1 @@ +{"epoch": 23.131672597864767, "train": {"Classification_binary": 0.802904486656189, "Classification": 0.00031463519553653896, "Regression_pt": 0.036622002720832825, "Regression_eta": 0.000207591918297112, "Regression_sin_phi": 0.00012786926527041942, "Regression_cos_phi": 9.109952225117013e-05, "Regression_energy": 0.036663368344306946, "Total": 1.7463117837905884, "Total_unweighted": 0.876931053622684}, "train_group": {}, "valid": {"Classification_binary": 1.1456964015960693, "Classification": 0.011242281645536423, "Regression_pt": 0.09986352175474167, "Regression_eta": 0.0010723366867750883, "Regression_sin_phi": 0.00906470324844122, "Regression_cos_phi": 0.004262850619852543, "Regression_energy": 0.10010989010334015, "Total": 14.574780464172363, "Total_unweighted": 1.3713119856547564, "metrics/particle/class_1/efficiency": 0.972540020942688, "metrics/particle/class_1/pid_accuracy": 0.9835293889045715, "metrics/particle/class_2/efficiency": 0.40740740299224854, "metrics/particle/class_2/pid_accuracy": 0.9090909361839294, "metrics/particle/class_3/efficiency": 0.8646464943885803, "metrics/particle/class_3/pid_accuracy": 0.9883177280426025, "metrics/particle/class_4/efficiency": 0.9491525292396545, "metrics/particle/class_4/pid_accuracy": 0.9375, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.01600000075995922, "metrics/particle/count/mae": 0.2240000069141388, "metrics/particle/count/prediction_mean": 4.631999969482422, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.041873060166835785, "metrics/particle/event/energy_response_mean": 1.002946376800537, "metrics/particle/event/met_abs_error": 1.0214331150054932, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04112032428383827, "metrics/particle/event/scalar_pt_response_mean": 1.0013972520828247, "metrics/particle/event/vector_pt_closure": 0.038974642753601074, "metrics/particle/matched/delta_eta_abs_mean": 0.006780073046684265, "metrics/particle/matched/delta_phi_abs_mean": 0.006518356967717409, "metrics/particle/matched/delta_r_mean": 0.010527814738452435, "metrics/particle/matched/energy_relative_abs_mean": 0.05977996811270714, "metrics/particle/matched/pid_accuracy": 0.9811320900917053, "metrics/particle/matched/pt_relative_abs_mean": 0.05979456380009651, "metrics/particle/matching/duplicate_fraction": 0.01295336801558733, "metrics/particle/matching/efficiency": 0.912220299243927, "metrics/particle/matching/f1": 0.9137930870056152, "metrics/particle/matching/purity": 0.9153713583946228, "metrics/particle/matching/target_coverage_dr0p05": 0.9070568084716797, "metrics/particle/matching/target_coverage_dr0p10": 0.9406196475028992, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8459552526473999, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9337349534034729}, "step_train_time": 0.4747164249420166, "step_valid_time": 2.8419495759999336, "step_total_time": 1527.5172953605652, "elapsed_training_loop_seconds": 1527.5172953605652, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9991600395017326, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02623513229857144, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9883449883449883, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.986046511627907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.011655011655011656, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.986013986013986, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9837209302325581, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9837587006960556, "step_test_time": 4.142775719999918, "step_plot_time": 3.4918279749999783, "step_validation_cycle_time": 10.791295221999917} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_14000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_14000.json new file mode 100644 index 0000000000000000000000000000000000000000..34de3d554fd3546ae58505523026477026098cfb --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_14000.json @@ -0,0 +1 @@ +{"epoch": 24.91103202846975, "train": {"Classification_binary": 0.36916598677635193, "Classification": 0.0009546341607347131, "Regression_pt": 0.12364885210990906, "Regression_eta": 0.002832392230629921, "Regression_sin_phi": 8.080144471023232e-05, "Regression_cos_phi": 0.000146579128340818, "Regression_energy": 0.12238351255655289, "Total": 3.676068067550659, "Total_unweighted": 0.6192127584072296}, "train_group": {}, "valid": {"Classification_binary": 1.1904504299163818, "Classification": 0.012163390405476093, "Regression_pt": 0.1057642251253128, "Regression_eta": 0.00098518468439579, "Regression_sin_phi": 0.009118837304413319, "Regression_cos_phi": 0.004345728084445, "Regression_energy": 0.10579580068588257, "Total": 14.888102531433105, "Total_unweighted": 1.4286235962063074, "metrics/particle/class_1/efficiency": 0.9748283624649048, "metrics/particle/class_1/pid_accuracy": 0.9835680723190308, "metrics/particle/class_2/efficiency": 0.4444444477558136, "metrics/particle/class_2/pid_accuracy": 0.9166666865348816, "metrics/particle/class_3/efficiency": 0.8747474551200867, "metrics/particle/class_3/pid_accuracy": 0.9907621145248413, "metrics/particle/class_4/efficiency": 0.9576271176338196, "metrics/particle/class_4/pid_accuracy": 0.9557521939277649, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.03200000151991844, "metrics/particle/count/mae": 0.2160000056028366, "metrics/particle/count/prediction_mean": 4.616000175476074, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04468946158885956, "metrics/particle/event/energy_response_mean": 0.9962151646614075, "metrics/particle/event/met_abs_error": 1.0477062463760376, "metrics/particle/event/scalar_pt_relative_abs_error": 0.043778542429208755, "metrics/particle/event/scalar_pt_response_mean": 0.9955706000328064, "metrics/particle/event/vector_pt_closure": 0.041014183312654495, "metrics/particle/matched/delta_eta_abs_mean": 0.007092291489243507, "metrics/particle/matched/delta_phi_abs_mean": 0.007095593027770519, "metrics/particle/matched/delta_r_mean": 0.011176705360412598, "metrics/particle/matched/energy_relative_abs_mean": 0.06206788867712021, "metrics/particle/matched/pid_accuracy": 0.9840824007987976, "metrics/particle/matched/pt_relative_abs_mean": 0.062069546431303024, "metrics/particle/matching/duplicate_fraction": 0.010398613288998604, "metrics/particle/matching/efficiency": 0.9191049933433533, "metrics/particle/matching/f1": 0.9222797751426697, "metrics/particle/matching/purity": 0.925476610660553, "metrics/particle/matching/target_coverage_dr0p05": 0.9027538895606995, "metrics/particle/matching/target_coverage_dr0p10": 0.9440619349479675, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8528399467468262, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9397590160369873}, "step_train_time": 0.46114516258239746, "step_valid_time": 2.8217993640000714, "step_total_time": 1645.1651182174683, "elapsed_training_loop_seconds": 1645.1651182174683, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9983836510032442, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.029908009792531498, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 432, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9884259259259259, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9895712630359212, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.011574074074074073, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9861111111111112, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9872537659327926, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.179410486000052, "step_plot_time": 3.59764115400003, "step_validation_cycle_time": 10.90996695900003} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_15000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_15000.json new file mode 100644 index 0000000000000000000000000000000000000000..269b959b6bef92d2d51956fee657fae0403cbfce --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_15000.json @@ -0,0 +1 @@ +{"epoch": 26.690391459074732, "train": {"Classification_binary": 0.14102520048618317, "Classification": 1.765978231560439e-05, "Regression_pt": 0.014123519882559776, "Regression_eta": 0.0002900639665313065, "Regression_sin_phi": 0.00015014983364380896, "Regression_cos_phi": 0.00013418668822851032, "Regression_energy": 0.013684889301657677, "Total": 0.6973263025283813, "Total_unweighted": 0.16942566994111985}, "train_group": {}, "valid": {"Classification_binary": 1.2549563646316528, "Classification": 0.013437874615192413, "Regression_pt": 0.11292235553264618, "Regression_eta": 0.001002920907922089, "Regression_sin_phi": 0.004983407445251942, "Regression_cos_phi": 0.002581679029390216, "Regression_energy": 0.11308223009109497, "Total": 10.62695598602295, "Total_unweighted": 1.5029668322531506, "metrics/particle/class_1/efficiency": 0.9794050455093384, "metrics/particle/class_1/pid_accuracy": 0.9813084006309509, "metrics/particle/class_2/efficiency": 0.5555555820465088, "metrics/particle/class_2/pid_accuracy": 1.0, "metrics/particle/class_3/efficiency": 0.8767676949501038, "metrics/particle/class_3/pid_accuracy": 0.9907833933830261, "metrics/particle/class_4/efficiency": 0.9406779408454895, "metrics/particle/class_4/pid_accuracy": 0.9639639854431152, "metrics/particle/class_5/efficiency": 0.9764705896377563, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": 0.0, "metrics/particle/count/mae": 0.25600001215934753, "metrics/particle/count/prediction_mean": 4.6479997634887695, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04440912976861, "metrics/particle/event/energy_response_mean": 0.9981765151023865, "metrics/particle/event/met_abs_error": 1.1056971549987793, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04310283437371254, "metrics/particle/event/scalar_pt_response_mean": 0.9985020756721497, "metrics/particle/event/vector_pt_closure": 0.04213736206293106, "metrics/particle/matched/delta_eta_abs_mean": 0.00697475578635931, "metrics/particle/matched/delta_phi_abs_mean": 0.006788857281208038, "metrics/particle/matched/delta_r_mean": 0.01087039615958929, "metrics/particle/matched/energy_relative_abs_mean": 0.06040915474295616, "metrics/particle/matched/pid_accuracy": 0.9850606918334961, "metrics/particle/matched/pt_relative_abs_mean": 0.06062600389122963, "metrics/particle/matching/duplicate_fraction": 0.014629947952926159, "metrics/particle/matching/efficiency": 0.9216867685317993, "metrics/particle/matching/f1": 0.9216867685317993, "metrics/particle/matching/purity": 0.9216867685317993, "metrics/particle/matching/target_coverage_dr0p05": 0.9027538895606995, "metrics/particle/matching/target_coverage_dr0p10": 0.9466437101364136, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8562822937965393, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9363166689872742}, "step_train_time": 0.4619772434234619, "step_valid_time": 2.8150969270000132, "step_total_time": 1762.8841607570648, "elapsed_training_loop_seconds": 1762.8841607570648, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9988067849892907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.028219189513769338, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9895470383275261, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.009302325581395349, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9895470383275261, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.221054862000074, "step_plot_time": 3.512213643999985, "step_validation_cycle_time": 10.869552395000028} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_16000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_16000.json new file mode 100644 index 0000000000000000000000000000000000000000..30dea153885f86ae543f2dcf4f24300be211ea82 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_16000.json @@ -0,0 +1 @@ +{"epoch": 28.469750889679716, "train": {"Classification_binary": 0.13585543632507324, "Classification": 3.825093699560966e-06, "Regression_pt": 0.011551782488822937, "Regression_eta": 0.0001483169908169657, "Regression_sin_phi": 2.5510980776743963e-05, "Regression_cos_phi": 0.00013969754218123853, "Regression_energy": 0.011618798598647118, "Total": 0.515627384185791, "Total_unweighted": 0.1593433680200178}, "train_group": {}, "valid": {"Classification_binary": 1.2792106866836548, "Classification": 0.0147221889346838, "Regression_pt": 0.1056608036160469, "Regression_eta": 0.0010849266545847058, "Regression_sin_phi": 0.0036865826696157455, "Regression_cos_phi": 0.002237604698166251, "Regression_energy": 0.10573869943618774, "Total": 9.358190536499023, "Total_unweighted": 1.51234149269294, "metrics/particle/class_1/efficiency": 0.9839816689491272, "metrics/particle/class_1/pid_accuracy": 0.9790697693824768, "metrics/particle/class_2/efficiency": 0.5185185074806213, "metrics/particle/class_2/pid_accuracy": 0.7857142686843872, "metrics/particle/class_3/efficiency": 0.8767676949501038, "metrics/particle/class_3/pid_accuracy": 0.9907833933830261, "metrics/particle/class_4/efficiency": 0.9322034120559692, "metrics/particle/class_4/pid_accuracy": 0.9636363387107849, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": 0.004000000189989805, "metrics/particle/count/mae": 0.24400000274181366, "metrics/particle/count/prediction_mean": 4.6519999504089355, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04401107877492905, "metrics/particle/event/energy_response_mean": 0.9980871677398682, "metrics/particle/event/met_abs_error": 1.1084049940109253, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04284033551812172, "metrics/particle/event/scalar_pt_response_mean": 0.9969161748886108, "metrics/particle/event/vector_pt_closure": 0.042390529066324234, "metrics/particle/matched/delta_eta_abs_mean": 0.007242686115205288, "metrics/particle/matched/delta_phi_abs_mean": 0.0061977277509868145, "metrics/particle/matched/delta_r_mean": 0.010603437200188637, "metrics/particle/matched/energy_relative_abs_mean": 0.06160799413919449, "metrics/particle/matched/pid_accuracy": 0.9813432693481445, "metrics/particle/matched/pt_relative_abs_mean": 0.06190497800707817, "metrics/particle/matching/duplicate_fraction": 0.012897678650915623, "metrics/particle/matching/efficiency": 0.9225473403930664, "metrics/particle/matching/f1": 0.9221505522727966, "metrics/particle/matching/purity": 0.9217540621757507, "metrics/particle/matching/target_coverage_dr0p05": 0.9104991555213928, "metrics/particle/matching/target_coverage_dr0p10": 0.9509466290473938, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8511187434196472, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9388984441757202}, "step_train_time": 0.46457362174987793, "step_valid_time": 2.8215337259998705, "step_total_time": 1881.417537689209, "elapsed_training_loop_seconds": 1881.417537689209, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9988571380833309, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.029730956847268675, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9883449883449883, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.986046511627907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.011655011655011656, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.986013986013986, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9837209302325581, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9837587006960556, "step_test_time": 4.196808094000062, "step_plot_time": 3.588732070000333, "step_validation_cycle_time": 10.930093295999995} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_17000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_17000.json new file mode 100644 index 0000000000000000000000000000000000000000..26dce4d498d1c465fb9b51f1cbcbc68e0de717de --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_17000.json @@ -0,0 +1 @@ +{"epoch": 30.249110320284696, "train": {"Classification_binary": 0.34019196033477783, "Classification": 3.444934554863721e-05, "Regression_pt": 0.029526468366384506, "Regression_eta": 0.00018592206470202655, "Regression_sin_phi": 0.00010202468547504395, "Regression_cos_phi": 0.00012568059901241213, "Regression_energy": 0.029754748567938805, "Total": 1.1259936094284058, "Total_unweighted": 0.39992125396383926}, "train_group": {}, "valid": {"Classification_binary": 1.3423622846603394, "Classification": 0.016682229936122894, "Regression_pt": 0.1051662415266037, "Regression_eta": 0.0009385799639858305, "Regression_sin_phi": 0.009501776657998562, "Regression_cos_phi": 0.004326163325458765, "Regression_energy": 0.10498052090406418, "Total": 15.684670448303223, "Total_unweighted": 1.5839577969745733, "metrics/particle/class_1/efficiency": 0.9794050455093384, "metrics/particle/class_1/pid_accuracy": 0.9906542301177979, "metrics/particle/class_2/efficiency": 0.5555555820465088, "metrics/particle/class_2/pid_accuracy": 0.8666666746139526, "metrics/particle/class_3/efficiency": 0.8666666746139526, "metrics/particle/class_3/pid_accuracy": 0.9930070042610168, "metrics/particle/class_4/efficiency": 0.9406779408454895, "metrics/particle/class_4/pid_accuracy": 0.9459459185600281, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": 0.00800000037997961, "metrics/particle/count/mae": 0.24799999594688416, "metrics/particle/count/prediction_mean": 4.656000137329102, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.042255181819200516, "metrics/particle/event/energy_response_mean": 1.0033622980117798, "metrics/particle/event/met_abs_error": 1.1415417194366455, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04114214703440666, "metrics/particle/event/scalar_pt_response_mean": 1.0024458169937134, "metrics/particle/event/vector_pt_closure": 0.040601152926683426, "metrics/particle/matched/delta_eta_abs_mean": 0.007259568199515343, "metrics/particle/matched/delta_phi_abs_mean": 0.006365555338561535, "metrics/particle/matched/delta_r_mean": 0.010677240788936615, "metrics/particle/matched/energy_relative_abs_mean": 0.06044907122850418, "metrics/particle/matched/pid_accuracy": 0.9859418869018555, "metrics/particle/matched/pt_relative_abs_mean": 0.060822807252407074, "metrics/particle/matching/duplicate_fraction": 0.017182130366563797, "metrics/particle/matching/efficiency": 0.9182444214820862, "metrics/particle/matching/f1": 0.9174548387527466, "metrics/particle/matching/purity": 0.9166666865348816, "metrics/particle/matching/target_coverage_dr0p05": 0.9104991555213928, "metrics/particle/matching/target_coverage_dr0p10": 0.9440619349479675, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8554216623306274, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9388984441757202}, "step_train_time": 0.46579766273498535, "step_valid_time": 2.8520950539996193, "step_total_time": 2000.5244615077972, "elapsed_training_loop_seconds": 2000.5244615077972, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9991190550437717, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.028989807339464546, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.006993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9906759906759907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.187735858999986, "step_plot_time": 3.550952142999904, "step_validation_cycle_time": 10.91654059199982} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_18000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_18000.json new file mode 100644 index 0000000000000000000000000000000000000000..457aff14d78831af35dbdcd7ba3606e09caa1573 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_18000.json @@ -0,0 +1 @@ +{"epoch": 32.02846975088968, "train": {"Classification_binary": 0.2563222646713257, "Classification": 0.0009616136667318642, "Regression_pt": 0.010062380693852901, "Regression_eta": 0.00010528725397307426, "Regression_sin_phi": 0.00011384037497919053, "Regression_cos_phi": 4.0712213376536965e-05, "Regression_energy": 0.01013426948338747, "Total": 0.6758066415786743, "Total_unweighted": 0.2777403683576267}, "train_group": {}, "valid": {"Classification_binary": 1.370851755142212, "Classification": 0.018095094710588455, "Regression_pt": 0.11626088619232178, "Regression_eta": 0.0009466364281252027, "Regression_sin_phi": 0.008516963571310043, "Regression_cos_phi": 0.004064691718667746, "Regression_energy": 0.11562405526638031, "Total": 15.066113471984863, "Total_unweighted": 1.6343600830296054, "metrics/particle/class_1/efficiency": 0.9816933870315552, "metrics/particle/class_1/pid_accuracy": 0.9813519716262817, "metrics/particle/class_2/efficiency": 0.48148149251937866, "metrics/particle/class_2/pid_accuracy": 0.8461538553237915, "metrics/particle/class_3/efficiency": 0.8767676949501038, "metrics/particle/class_3/pid_accuracy": 0.9907833933830261, "metrics/particle/class_4/efficiency": 0.9491525292396545, "metrics/particle/class_4/pid_accuracy": 0.9464285969734192, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": 0.004000000189989805, "metrics/particle/count/mae": 0.25200000405311584, "metrics/particle/count/prediction_mean": 4.6519999504089355, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.045740172266960144, "metrics/particle/event/energy_response_mean": 0.9989109635353088, "metrics/particle/event/met_abs_error": 1.2095184326171875, "metrics/particle/event/scalar_pt_relative_abs_error": 0.045473601669073105, "metrics/particle/event/scalar_pt_response_mean": 0.9982138872146606, "metrics/particle/event/vector_pt_closure": 0.045513223856687546, "metrics/particle/matched/delta_eta_abs_mean": 0.006538298912346363, "metrics/particle/matched/delta_phi_abs_mean": 0.006862844806164503, "metrics/particle/matched/delta_r_mean": 0.010586529970169067, "metrics/particle/matched/energy_relative_abs_mean": 0.061815887689590454, "metrics/particle/matched/pid_accuracy": 0.9813432693481445, "metrics/particle/matched/pt_relative_abs_mean": 0.061851657927036285, "metrics/particle/matching/duplicate_fraction": 0.012897678650915623, "metrics/particle/matching/efficiency": 0.9225473403930664, "metrics/particle/matching/f1": 0.9221505522727966, "metrics/particle/matching/purity": 0.9217540621757507, "metrics/particle/matching/target_coverage_dr0p05": 0.9113597273826599, "metrics/particle/matching/target_coverage_dr0p10": 0.9449225664138794, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8562822937965393, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9414802193641663}, "step_train_time": 0.46262598037719727, "step_valid_time": 2.8418200559999605, "step_total_time": 2119.6766517162323, "elapsed_training_loop_seconds": 2119.6766517162323, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.999827043059946, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.029783712671293427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 422, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.006993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9791183294663574, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9836829836829837, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9813953488372092, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.22873325799992, "step_plot_time": 3.5189142790000005, "step_validation_cycle_time": 10.91482783299989} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_19000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_19000.json new file mode 100644 index 0000000000000000000000000000000000000000..681d13c4d7077579d1c832b6ac832a19b692638c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_19000.json @@ -0,0 +1 @@ +{"epoch": 33.80782918149466, "train": {"Classification_binary": 0.6686391830444336, "Classification": 0.00042082788422703743, "Regression_pt": 0.01929880678653717, "Regression_eta": 0.0008440539240837097, "Regression_sin_phi": 0.00012474137474782765, "Regression_cos_phi": 8.469842578051612e-05, "Regression_energy": 0.017921576276421547, "Total": 1.4325921535491943, "Total_unweighted": 0.7073338877162314}, "train_group": {}, "valid": {"Classification_binary": 1.434572696685791, "Classification": 0.01789177395403385, "Regression_pt": 0.10571164637804031, "Regression_eta": 0.0010746499756351113, "Regression_sin_phi": 0.006863016169518232, "Regression_cos_phi": 0.0032185774762183428, "Regression_energy": 0.10586953163146973, "Total": 13.020676612854004, "Total_unweighted": 1.6752018922707066, "metrics/particle/class_1/efficiency": 0.972540020942688, "metrics/particle/class_1/pid_accuracy": 0.9835293889045715, "metrics/particle/class_2/efficiency": 0.4444444477558136, "metrics/particle/class_2/pid_accuracy": 0.9166666865348816, "metrics/particle/class_3/efficiency": 0.8666666746139526, "metrics/particle/class_3/pid_accuracy": 0.9883449673652649, "metrics/particle/class_4/efficiency": 0.9491525292396545, "metrics/particle/class_4/pid_accuracy": 0.9375, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": 0.0, "metrics/particle/count/mae": 0.2639999985694885, "metrics/particle/count/prediction_mean": 4.6479997634887695, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04517746344208717, "metrics/particle/event/energy_response_mean": 1.0051944255828857, "metrics/particle/event/met_abs_error": 1.1772029399871826, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04493231326341629, "metrics/particle/event/scalar_pt_response_mean": 1.0053958892822266, "metrics/particle/event/vector_pt_closure": 0.04432220384478569, "metrics/particle/matched/delta_eta_abs_mean": 0.007042350247502327, "metrics/particle/matched/delta_phi_abs_mean": 0.006391473114490509, "metrics/particle/matched/delta_r_mean": 0.010566271841526031, "metrics/particle/matched/energy_relative_abs_mean": 0.060713667422533035, "metrics/particle/matched/pid_accuracy": 0.9811676144599915, "metrics/particle/matched/pt_relative_abs_mean": 0.061065930873155594, "metrics/particle/matching/duplicate_fraction": 0.016351118683815002, "metrics/particle/matching/efficiency": 0.913941502571106, "metrics/particle/matching/f1": 0.913941502571106, "metrics/particle/matching/purity": 0.913941502571106, "metrics/particle/matching/target_coverage_dr0p05": 0.9036144614219666, "metrics/particle/matching/target_coverage_dr0p10": 0.9397590160369873, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8476763963699341, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.93459552526474}, "step_train_time": 0.45540523529052734, "step_valid_time": 2.8045250380000653, "step_total_time": 2238.5160875320435, "elapsed_training_loop_seconds": 2238.5160875320435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0032143979951615, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.029614770765491127, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.986046511627907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9849012775842044, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.013953488372093023, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9837209302325581, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9825783972125436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9837587006960556, "step_test_time": 4.133830315000523, "step_plot_time": 3.55921508099982, "step_validation_cycle_time": 10.818126082999697} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_2000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_2000.json new file mode 100644 index 0000000000000000000000000000000000000000..d458693be00ab24ed4de17b09f7444a35aab8944 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_2000.json @@ -0,0 +1 @@ +{"epoch": 3.5587188612099645, "train": {"Classification_binary": 0.833330512046814, "Classification": 0.01035182923078537, "Regression_pt": 0.03788209706544876, "Regression_eta": 0.000229576020501554, "Regression_sin_phi": 0.00022414224804379046, "Regression_cos_phi": 9.616257739253342e-05, "Regression_energy": 0.03746384009718895, "Total": 2.7241482734680176, "Total_unweighted": 0.9195781592861749}, "train_group": {}, "valid": {"Classification_binary": 1.0779540538787842, "Classification": 0.00905871856957674, "Regression_pt": 0.09698648005723953, "Regression_eta": 0.0010703729931265116, "Regression_sin_phi": 0.005634731147438288, "Regression_cos_phi": 0.0024769131559878588, "Regression_energy": 0.09751126170158386, "Total": 10.243151664733887, "Total_unweighted": 1.290692531503737, "metrics/particle/class_1/efficiency": 0.972540020942688, "metrics/particle/class_1/pid_accuracy": 0.9882352948188782, "metrics/particle/class_2/efficiency": 0.6296296119689941, "metrics/particle/class_2/pid_accuracy": 0.9411764740943909, "metrics/particle/class_3/efficiency": 0.8666666746139526, "metrics/particle/class_3/pid_accuracy": 0.997668981552124, "metrics/particle/class_4/efficiency": 0.9237288236618042, "metrics/particle/class_4/pid_accuracy": 0.963302731513977, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 0.988095223903656, "metrics/particle/count/bias_mean": -0.03999999910593033, "metrics/particle/count/mae": 0.25600001215934753, "metrics/particle/count/prediction_mean": 4.607999801635742, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04598569869995117, "metrics/particle/event/energy_response_mean": 0.995384693145752, "metrics/particle/event/met_abs_error": 1.0707101821899414, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04529513046145439, "metrics/particle/event/scalar_pt_response_mean": 0.9961782693862915, "metrics/particle/event/vector_pt_closure": 0.042002204805612564, "metrics/particle/matched/delta_eta_abs_mean": 0.007841896265745163, "metrics/particle/matched/delta_phi_abs_mean": 0.006771971471607685, "metrics/particle/matched/delta_r_mean": 0.01153208501636982, "metrics/particle/matched/energy_relative_abs_mean": 0.062033459544181824, "metrics/particle/matched/pid_accuracy": 0.9887217879295349, "metrics/particle/matched/pt_relative_abs_mean": 0.06255573779344559, "metrics/particle/matching/duplicate_fraction": 0.0173611119389534, "metrics/particle/matching/efficiency": 0.9156626462936401, "metrics/particle/matching/f1": 0.9196196794509888, "metrics/particle/matching/purity": 0.9236111044883728, "metrics/particle/matching/target_coverage_dr0p05": 0.9113597273826599, "metrics/particle/matching/target_coverage_dr0p10": 0.9466437101364136, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8416523337364197, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9302926063537598}, "step_train_time": 0.4462902545928955, "step_valid_time": 2.8247715960001187, "step_total_time": 229.64044761657715, "elapsed_training_loop_seconds": 229.64044761657715, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001990721727276, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.029993859890646957, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9930232558139535, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.991869918699187, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0069767441860465115, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9895470383275261, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.089505374000055, "step_plot_time": 3.469646709999779, "step_validation_cycle_time": 10.691038454000136} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..ba4c8227c44f6ac647551d71f4e6267fd937e0fa --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 0.08239409327507019, "Classification": 1.4354060112964362e-05, "Regression_pt": 0.022041667252779007, "Regression_eta": 8.457192598143592e-05, "Regression_sin_phi": 0.0001255181123269722, "Regression_cos_phi": 7.71630511735566e-05, "Regression_energy": 0.02148931659758091, "Total": 0.6747820973396301, "Total_unweighted": 0.12622668427502504}, "train_group": {}, "valid": {"Classification_binary": 1.5324878692626953, "Classification": 0.017693761736154556, "Regression_pt": 0.10949704796075821, "Regression_eta": 0.0009850068017840385, "Regression_sin_phi": 0.006993222516030073, "Regression_cos_phi": 0.00320238946005702, "Regression_energy": 0.10956847667694092, "Total": 13.242388725280762, "Total_unweighted": 1.7804277744144201, "metrics/particle/class_1/efficiency": 0.9748283624649048, "metrics/particle/class_1/pid_accuracy": 0.9835680723190308, "metrics/particle/class_2/efficiency": 0.4444444477558136, "metrics/particle/class_2/pid_accuracy": 0.9166666865348816, "metrics/particle/class_3/efficiency": 0.8767676949501038, "metrics/particle/class_3/pid_accuracy": 0.9861751198768616, "metrics/particle/class_4/efficiency": 0.9491525292396545, "metrics/particle/class_4/pid_accuracy": 0.9642857313156128, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": 0.004000000189989805, "metrics/particle/count/mae": 0.2680000066757202, "metrics/particle/count/prediction_mean": 4.6519999504089355, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04487406089901924, "metrics/particle/event/energy_response_mean": 0.9994185566902161, "metrics/particle/event/met_abs_error": 1.1750080585479736, "metrics/particle/event/scalar_pt_relative_abs_error": 0.043330851942300797, "metrics/particle/event/scalar_pt_response_mean": 0.9982456564903259, "metrics/particle/event/vector_pt_closure": 0.04294461011886597, "metrics/particle/matched/delta_eta_abs_mean": 0.0071101742796599865, "metrics/particle/matched/delta_phi_abs_mean": 0.006327133625745773, "metrics/particle/matched/delta_r_mean": 0.010614489205181599, "metrics/particle/matched/energy_relative_abs_mean": 0.05974479764699936, "metrics/particle/matched/pid_accuracy": 0.983146071434021, "metrics/particle/matched/pt_relative_abs_mean": 0.05990730971097946, "metrics/particle/matching/duplicate_fraction": 0.013757524080574512, "metrics/particle/matching/efficiency": 0.9191049933433533, "metrics/particle/matching/f1": 0.9187096953392029, "metrics/particle/matching/purity": 0.9183146953582764, "metrics/particle/matching/target_coverage_dr0p05": 0.9027538895606995, "metrics/particle/matching/target_coverage_dr0p10": 0.9397590160369873, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8528399467468262, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9397590160369873}, "step_train_time": 0.4624145030975342, "step_valid_time": 2.8496266929996636, "step_total_time": 2356.9669353961945, "elapsed_training_loop_seconds": 2356.9669353961945, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0007543254640965, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.031098222450793855, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.01160092807424594, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.1665758910003206, "step_plot_time": 3.541614151000431, "step_validation_cycle_time": 10.883029019000787} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_21000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_21000.json new file mode 100644 index 0000000000000000000000000000000000000000..939b5ccbcc728c4252427463d140d75f323d23de --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_21000.json @@ -0,0 +1 @@ +{"epoch": 37.36654804270463, "train": {"Classification_binary": 0.8259889483451843, "Classification": 1.5392404748126864e-05, "Regression_pt": 0.0077852304093539715, "Regression_eta": 0.00011696595174726099, "Regression_sin_phi": 3.780047336476855e-05, "Regression_cos_phi": 8.447487925877795e-05, "Regression_energy": 0.008145439438521862, "Total": 1.099662184715271, "Total_unweighted": 0.8421742519021791}, "train_group": {}, "valid": {"Classification_binary": 1.6370965242385864, "Classification": 0.021675661206245422, "Regression_pt": 0.11308367550373077, "Regression_eta": 0.0010530634317547083, "Regression_sin_phi": 0.009655059315264225, "Regression_cos_phi": 0.0046027773059904575, "Regression_energy": 0.11302141845226288, "Total": 16.901653289794922, "Total_unweighted": 1.900188179453835, "metrics/particle/class_1/efficiency": 0.9771167039871216, "metrics/particle/class_1/pid_accuracy": 0.9859485030174255, "metrics/particle/class_2/efficiency": 0.48148149251937866, "metrics/particle/class_2/pid_accuracy": 0.9230769276618958, "metrics/particle/class_3/efficiency": 0.8787878751754761, "metrics/particle/class_3/pid_accuracy": 0.9862068891525269, "metrics/particle/class_4/efficiency": 0.9491525292396545, "metrics/particle/class_4/pid_accuracy": 0.9375, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": 0.024000000208616257, "metrics/particle/count/mae": 0.2800000011920929, "metrics/particle/count/prediction_mean": 4.671999931335449, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.045564498752355576, "metrics/particle/event/energy_response_mean": 1.0005888938903809, "metrics/particle/event/met_abs_error": 1.1194366216659546, "metrics/particle/event/scalar_pt_relative_abs_error": 0.044500693678855896, "metrics/particle/event/scalar_pt_response_mean": 1.00007963180542, "metrics/particle/event/vector_pt_closure": 0.04188676178455353, "metrics/particle/matched/delta_eta_abs_mean": 0.007240612991154194, "metrics/particle/matched/delta_phi_abs_mean": 0.006073407828807831, "metrics/particle/matched/delta_r_mean": 0.010575731284916401, "metrics/particle/matched/energy_relative_abs_mean": 0.06344307214021683, "metrics/particle/matched/pid_accuracy": 0.9813258647918701, "metrics/particle/matched/pt_relative_abs_mean": 0.06356877088546753, "metrics/particle/matching/duplicate_fraction": 0.013698630034923553, "metrics/particle/matching/efficiency": 0.9216867685317993, "metrics/particle/matching/f1": 0.9193133115768433, "metrics/particle/matching/purity": 0.9169520735740662, "metrics/particle/matching/target_coverage_dr0p05": 0.9036144614219666, "metrics/particle/matching/target_coverage_dr0p10": 0.9423407912254333, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.848537027835846, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9432013630867004}, "step_train_time": 0.46492576599121094, "step_valid_time": 2.810518690000208, "step_total_time": 2475.3767800331116, "elapsed_training_loop_seconds": 2475.3767800331116, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9994423430153042, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.030240216901079253, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9906759906759907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.009324009324009324, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.986013986013986, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9837209302325581, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9860788863109049, "step_test_time": 4.214345295000385, "step_plot_time": 3.549170413999491, "step_validation_cycle_time": 10.913087906999863} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_22000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_22000.json new file mode 100644 index 0000000000000000000000000000000000000000..6a2e92bc62fe01d34285137fee4dae3b3e738cb6 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_22000.json @@ -0,0 +1 @@ +{"epoch": 39.145907473309606, "train": {"Classification_binary": 0.05775713175535202, "Classification": 7.1863364610180724e-06, "Regression_pt": 0.01119777001440525, "Regression_eta": 0.00026401394279673696, "Regression_sin_phi": 6.818043038947508e-05, "Regression_cos_phi": 0.00010254468361381441, "Regression_energy": 0.012064261361956596, "Total": 0.4749133884906769, "Total_unweighted": 0.08146108852497491}, "train_group": {}, "valid": {"Classification_binary": 1.6721071004867554, "Classification": 0.02071111649274826, "Regression_pt": 0.1147257462143898, "Regression_eta": 0.0009370908373966813, "Regression_sin_phi": 0.008507486432790756, "Regression_cos_phi": 0.0040921433828771114, "Regression_energy": 0.11488115042448044, "Total": 15.574950218200684, "Total_unweighted": 1.9359618342714384, "metrics/particle/class_1/efficiency": 0.972540020942688, "metrics/particle/class_1/pid_accuracy": 0.9882352948188782, "metrics/particle/class_2/efficiency": 0.48148149251937866, "metrics/particle/class_2/pid_accuracy": 0.9230769276618958, "metrics/particle/class_3/efficiency": 0.8585858345031738, "metrics/particle/class_3/pid_accuracy": 0.9882352948188782, "metrics/particle/class_4/efficiency": 0.9406779408454895, "metrics/particle/class_4/pid_accuracy": 0.9639639854431152, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.0560000017285347, "metrics/particle/count/mae": 0.2639999985694885, "metrics/particle/count/prediction_mean": 4.5920000076293945, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.044655367732048035, "metrics/particle/event/energy_response_mean": 0.99770587682724, "metrics/particle/event/met_abs_error": 1.2172753810882568, "metrics/particle/event/scalar_pt_relative_abs_error": 0.043397534638643265, "metrics/particle/event/scalar_pt_response_mean": 0.9960697293281555, "metrics/particle/event/vector_pt_closure": 0.046314582228660583, "metrics/particle/matched/delta_eta_abs_mean": 0.0071892342530190945, "metrics/particle/matched/delta_phi_abs_mean": 0.006933210417628288, "metrics/particle/matched/delta_r_mean": 0.011080439202487469, "metrics/particle/matched/energy_relative_abs_mean": 0.06083529815077782, "metrics/particle/matched/pid_accuracy": 0.985822319984436, "metrics/particle/matched/pt_relative_abs_mean": 0.060934703797101974, "metrics/particle/matching/duplicate_fraction": 0.012195121496915817, "metrics/particle/matching/efficiency": 0.9104991555213928, "metrics/particle/matching/f1": 0.9160172939300537, "metrics/particle/matching/purity": 0.9216027855873108, "metrics/particle/matching/target_coverage_dr0p05": 0.8984509706497192, "metrics/particle/matching/target_coverage_dr0p10": 0.9328743815422058, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8511187434196472, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9302926063537598}, "step_train_time": 0.46391963958740234, "step_valid_time": 2.8067396320002445, "step_total_time": 2593.980874300003, "elapsed_training_loop_seconds": 2593.980874300003, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0003932128155386, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.030476087870123125, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9872241579558653, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.011627906976744186, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.986046511627907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9849012775842044, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9860788863109049, "step_test_time": 4.226678887999697, "step_plot_time": 3.5956937709997874, "step_validation_cycle_time": 10.952259351000066} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_23000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_23000.json new file mode 100644 index 0000000000000000000000000000000000000000..da863b0d38ae347458b10e136f2e8df3c212bd95 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_23000.json @@ -0,0 +1 @@ +{"epoch": 40.92526690391459, "train": {"Classification_binary": 0.2970917522907257, "Classification": 1.2463670827855822e-05, "Regression_pt": 0.02768520638346672, "Regression_eta": 0.00021906108304392546, "Regression_sin_phi": 0.0001674744999036193, "Regression_cos_phi": 0.00016177094948943704, "Regression_energy": 0.026241572573781013, "Total": 1.1171454191207886, "Total_unweighted": 0.3515793014512383}, "train_group": {}, "valid": {"Classification_binary": 1.7430925369262695, "Classification": 0.023843299597501755, "Regression_pt": 0.12144137918949127, "Regression_eta": 0.0010316282277926803, "Regression_sin_phi": 0.01058483961969614, "Regression_cos_phi": 0.0052485899068415165, "Regression_energy": 0.12201930582523346, "Total": 18.558101654052734, "Total_unweighted": 2.0272615792928264, "metrics/particle/class_1/efficiency": 0.9679633975028992, "metrics/particle/class_1/pid_accuracy": 0.9858155846595764, "metrics/particle/class_2/efficiency": 0.5185185074806213, "metrics/particle/class_2/pid_accuracy": 0.8571428656578064, "metrics/particle/class_3/efficiency": 0.8585858345031738, "metrics/particle/class_3/pid_accuracy": 0.9882352948188782, "metrics/particle/class_4/efficiency": 0.9491525292396545, "metrics/particle/class_4/pid_accuracy": 0.9464285969734192, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 0.988095223903656, "metrics/particle/count/bias_mean": -0.03200000151991844, "metrics/particle/count/mae": 0.2720000147819519, "metrics/particle/count/prediction_mean": 4.616000175476074, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.046944741159677505, "metrics/particle/event/energy_response_mean": 0.9948938488960266, "metrics/particle/event/met_abs_error": 1.2471023797988892, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04702312871813774, "metrics/particle/event/scalar_pt_response_mean": 0.9950293302536011, "metrics/particle/event/vector_pt_closure": 0.04656180366873741, "metrics/particle/matched/delta_eta_abs_mean": 0.007364633027464151, "metrics/particle/matched/delta_phi_abs_mean": 0.006116007920354605, "metrics/particle/matched/delta_r_mean": 0.010599135421216488, "metrics/particle/matched/energy_relative_abs_mean": 0.05923449993133545, "metrics/particle/matched/pid_accuracy": 0.9810963869094849, "metrics/particle/matched/pt_relative_abs_mean": 0.05949759855866432, "metrics/particle/matching/duplicate_fraction": 0.013864818029105663, "metrics/particle/matching/efficiency": 0.9104991555213928, "metrics/particle/matching/f1": 0.9136441946029663, "metrics/particle/matching/purity": 0.9168111085891724, "metrics/particle/matching/target_coverage_dr0p05": 0.9036144614219666, "metrics/particle/matching/target_coverage_dr0p10": 0.9397590160369873, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8519793748855591, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9294320344924927}, "step_train_time": 0.45858097076416016, "step_valid_time": 2.8560996969999906, "step_total_time": 2712.8424928188324, "elapsed_training_loop_seconds": 2712.8424928188324, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9964711837943641, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.031927272446688804, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.006993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.986013986013986, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9837209302325581, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.196326281999973, "step_plot_time": 3.522271945000284, "step_validation_cycle_time": 10.90236614999958} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_24000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_24000.json new file mode 100644 index 0000000000000000000000000000000000000000..e5397d54d5fe037257e18ca53c7801123e62950f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_24000.json @@ -0,0 +1 @@ +{"epoch": 42.704626334519574, "train": {"Classification_binary": 0.43954190611839294, "Classification": 1.842903293436393e-05, "Regression_pt": 0.020645728334784508, "Regression_eta": 0.0002669113455340266, "Regression_sin_phi": 0.00011161122529301792, "Regression_cos_phi": 8.839683869155124e-05, "Regression_energy": 0.0205401461571455, "Total": 1.053196668624878, "Total_unweighted": 0.4812131290527759}, "train_group": {}, "valid": {"Classification_binary": 1.7858812808990479, "Classification": 0.02130643092095852, "Regression_pt": 0.11373928934335709, "Regression_eta": 0.0010239389957860112, "Regression_sin_phi": 0.009458141401410103, "Regression_cos_phi": 0.004527070559561253, "Regression_energy": 0.11429042369127274, "Total": 16.818483352661133, "Total_unweighted": 2.0502265758113936, "metrics/particle/class_1/efficiency": 0.970251739025116, "metrics/particle/class_1/pid_accuracy": 0.9764150977134705, "metrics/particle/class_2/efficiency": 0.40740740299224854, "metrics/particle/class_2/pid_accuracy": 1.0, "metrics/particle/class_3/efficiency": 0.8787878751754761, "metrics/particle/class_3/pid_accuracy": 0.9908046126365662, "metrics/particle/class_4/efficiency": 0.9491525292396545, "metrics/particle/class_4/pid_accuracy": 0.9732142686843872, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.019999999552965164, "metrics/particle/count/mae": 0.2840000092983246, "metrics/particle/count/prediction_mean": 4.627999782562256, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04350413382053375, "metrics/particle/event/energy_response_mean": 0.9981811046600342, "metrics/particle/event/met_abs_error": 1.1692781448364258, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04440861940383911, "metrics/particle/event/scalar_pt_response_mean": 0.9958654642105103, "metrics/particle/event/vector_pt_closure": 0.046351317316293716, "metrics/particle/matched/delta_eta_abs_mean": 0.0070932949893176556, "metrics/particle/matched/delta_phi_abs_mean": 0.006845809053629637, "metrics/particle/matched/delta_r_mean": 0.010952788405120373, "metrics/particle/matched/energy_relative_abs_mean": 0.06107316166162491, "metrics/particle/matched/pid_accuracy": 0.9840525388717651, "metrics/particle/matched/pt_relative_abs_mean": 0.0612071268260479, "metrics/particle/matching/duplicate_fraction": 0.016421779990196228, "metrics/particle/matching/efficiency": 0.9173838496208191, "metrics/particle/matching/f1": 0.9193617701530457, "metrics/particle/matching/purity": 0.9213483333587646, "metrics/particle/matching/target_coverage_dr0p05": 0.9018932580947876, "metrics/particle/matching/target_coverage_dr0p10": 0.9388984441757202, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8511187434196472, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.93459552526474}, "step_train_time": 0.4623904228210449, "step_valid_time": 2.809117144999618, "step_total_time": 2831.635810852051, "elapsed_training_loop_seconds": 2831.635810852051, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9993447060499596, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02840461392899618, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9930232558139535, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.991869918699187, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0069767441860465115, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.986046511627907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9849012775842044, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.289268561000426, "step_plot_time": 3.5563144020006803, "step_validation_cycle_time": 10.986354809000659} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_25000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_25000.json new file mode 100644 index 0000000000000000000000000000000000000000..a759aa1c26f2c21692558b5af7500b1b026a4f84 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_25000.json @@ -0,0 +1 @@ +{"epoch": 44.48398576512456, "train": {"Classification_binary": 0.40490061044692993, "Classification": 6.108130037318915e-07, "Regression_pt": 0.0652480497956276, "Regression_eta": 0.00037246759166009724, "Regression_sin_phi": 8.971067290985957e-05, "Regression_cos_phi": 0.00022985121177043766, "Regression_energy": 0.06509196758270264, "Total": 1.9762637615203857, "Total_unweighted": 0.5359332681146043}, "train_group": {}, "valid": {"Classification_binary": 1.9926501512527466, "Classification": 0.02277532033622265, "Regression_pt": 0.11888858675956726, "Regression_eta": 0.0010711833601817489, "Regression_sin_phi": 0.005741266533732414, "Regression_cos_phi": 0.0028995478060096502, "Regression_energy": 0.11985349655151367, "Total": 13.120363235473633, "Total_unweighted": 2.263879552599974, "metrics/particle/class_1/efficiency": 0.972540020942688, "metrics/particle/class_1/pid_accuracy": 0.9858823418617249, "metrics/particle/class_2/efficiency": 0.48148149251937866, "metrics/particle/class_2/pid_accuracy": 1.0, "metrics/particle/class_3/efficiency": 0.8727272748947144, "metrics/particle/class_3/pid_accuracy": 0.9907407164573669, "metrics/particle/class_4/efficiency": 0.9491525292396545, "metrics/particle/class_4/pid_accuracy": 0.9553571343421936, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": 0.024000000208616257, "metrics/particle/count/mae": 0.2800000011920929, "metrics/particle/count/prediction_mean": 4.671999931335449, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.045761968940496445, "metrics/particle/event/energy_response_mean": 1.0009396076202393, "metrics/particle/event/met_abs_error": 1.2465182542800903, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04525087773799896, "metrics/particle/event/scalar_pt_response_mean": 1.0002825260162354, "metrics/particle/event/vector_pt_closure": 0.04634630307555199, "metrics/particle/matched/delta_eta_abs_mean": 0.006723085884004831, "metrics/particle/matched/delta_phi_abs_mean": 0.006619556806981564, "metrics/particle/matched/delta_r_mean": 0.010467533022165298, "metrics/particle/matched/energy_relative_abs_mean": 0.05903983488678932, "metrics/particle/matched/pid_accuracy": 0.9859287142753601, "metrics/particle/matched/pt_relative_abs_mean": 0.05936967581510544, "metrics/particle/matching/duplicate_fraction": 0.015410958789288998, "metrics/particle/matching/efficiency": 0.9173838496208191, "metrics/particle/matching/f1": 0.9150214791297913, "metrics/particle/matching/purity": 0.9126712083816528, "metrics/particle/matching/target_coverage_dr0p05": 0.9113597273826599, "metrics/particle/matching/target_coverage_dr0p10": 0.9440619349479675, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8562822937965393, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.93459552526474}, "step_train_time": 0.5556936264038086, "step_valid_time": 2.812816756000757, "step_total_time": 2951.4228377342224, "elapsed_training_loop_seconds": 2951.4228377342224, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9999226580701001, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.031140255930083627, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9883449883449883, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.986046511627907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.011655011655011656, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.986013986013986, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9837209302325581, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9837587006960556, "step_test_time": 4.2237114630006545, "step_plot_time": 3.5136336390005454, "step_validation_cycle_time": 10.871712773000581} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_26000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_26000.json new file mode 100644 index 0000000000000000000000000000000000000000..f823cd5d45cb84b8da83a4379525b067e02bbd7d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_26000.json @@ -0,0 +1 @@ +{"epoch": 46.263345195729535, "train": {"Classification_binary": 0.055816784501075745, "Classification": 1.1650729902612511e-05, "Regression_pt": 0.014857593923807144, "Regression_eta": 9.05164415598847e-05, "Regression_sin_phi": 8.695396536495537e-05, "Regression_cos_phi": 0.00012045416951877996, "Regression_energy": 0.015446750447154045, "Total": 0.5242971777915955, "Total_unweighted": 0.08643070417838317}, "train_group": {}, "valid": {"Classification_binary": 2.076432466506958, "Classification": 0.025436891242861748, "Regression_pt": 0.13570758700370789, "Regression_eta": 0.001062220660969615, "Regression_sin_phi": 0.010138486512005329, "Regression_cos_phi": 0.004936553072184324, "Regression_energy": 0.1356678307056427, "Total": 18.716257095336914, "Total_unweighted": 2.3893820357043296, "metrics/particle/class_1/efficiency": 0.9679633975028992, "metrics/particle/class_1/pid_accuracy": 0.9858155846595764, "metrics/particle/class_2/efficiency": 0.40740740299224854, "metrics/particle/class_2/pid_accuracy": 0.7272727489471436, "metrics/particle/class_3/efficiency": 0.8828282952308655, "metrics/particle/class_3/pid_accuracy": 0.986270010471344, "metrics/particle/class_4/efficiency": 0.9576271176338196, "metrics/particle/class_4/pid_accuracy": 0.9469026327133179, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": 0.02800000086426735, "metrics/particle/count/mae": 0.2840000092983246, "metrics/particle/count/prediction_mean": 4.676000118255615, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.049355994910001755, "metrics/particle/event/energy_response_mean": 1.0067111253738403, "metrics/particle/event/met_abs_error": 1.2899771928787231, "metrics/particle/event/scalar_pt_relative_abs_error": 0.048086557537317276, "metrics/particle/event/scalar_pt_response_mean": 1.0052675008773804, "metrics/particle/event/vector_pt_closure": 0.04790817201137543, "metrics/particle/matched/delta_eta_abs_mean": 0.007154601626098156, "metrics/particle/matched/delta_phi_abs_mean": 0.006768572609871626, "metrics/particle/matched/delta_r_mean": 0.010957794263958931, "metrics/particle/matched/energy_relative_abs_mean": 0.06055839732289314, "metrics/particle/matched/pid_accuracy": 0.9803370833396912, "metrics/particle/matched/pt_relative_abs_mean": 0.06098586693406105, "metrics/particle/matching/duplicate_fraction": 0.014542344026267529, "metrics/particle/matching/efficiency": 0.9191049933433533, "metrics/particle/matching/f1": 0.916344940662384, "metrics/particle/matching/purity": 0.9136013984680176, "metrics/particle/matching/target_coverage_dr0p05": 0.9130808711051941, "metrics/particle/matching/target_coverage_dr0p10": 0.9432013630867004, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8511187434196472, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9363166689872742}, "step_train_time": 0.4733402729034424, "step_valid_time": 2.848715948000063, "step_total_time": 3070.6844370365143, "elapsed_training_loop_seconds": 3070.6844370365143, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002189249729681, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.030913204202841604, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 422, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9906759906759907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.009324009324009324, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9791183294663574, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9836829836829837, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9813953488372092, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9860788863109049, "step_test_time": 4.212849010999889, "step_plot_time": 3.5824456219997955, "step_validation_cycle_time": 10.973240805999922} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_27000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_27000.json new file mode 100644 index 0000000000000000000000000000000000000000..0c2eb6159a575775a2ae11d1ea7d6d0e7d1f817b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_27000.json @@ -0,0 +1 @@ +{"epoch": 48.04270462633452, "train": {"Classification_binary": 0.11818581819534302, "Classification": 1.0150326943403343e-06, "Regression_pt": 0.011800292879343033, "Regression_eta": 0.00019379809964448214, "Regression_sin_phi": 9.972283442039043e-05, "Regression_cos_phi": 0.00015322552644647658, "Regression_energy": 0.011907613836228848, "Total": 0.5832829475402832, "Total_unweighted": 0.1423414864041206}, "train_group": {}, "valid": {"Classification_binary": 2.021423816680908, "Classification": 0.025698481127619743, "Regression_pt": 0.14061778783798218, "Regression_eta": 0.0009664525277912617, "Regression_sin_phi": 0.009574754163622856, "Regression_cos_phi": 0.0046881805174052715, "Regression_energy": 0.14147421717643738, "Total": 18.13133430480957, "Total_unweighted": 2.344443690031767, "metrics/particle/class_1/efficiency": 0.9656750559806824, "metrics/particle/class_1/pid_accuracy": 0.9834123253822327, "metrics/particle/class_2/efficiency": 0.4444444477558136, "metrics/particle/class_2/pid_accuracy": 0.8333333134651184, "metrics/particle/class_3/efficiency": 0.868686854839325, "metrics/particle/class_3/pid_accuracy": 0.9906976819038391, "metrics/particle/class_4/efficiency": 0.9491525292396545, "metrics/particle/class_4/pid_accuracy": 0.9732142686843872, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9647058844566345, "metrics/particle/count/bias_mean": 0.0, "metrics/particle/count/mae": 0.2879999876022339, "metrics/particle/count/prediction_mean": 4.6479997634887695, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04427380859851837, "metrics/particle/event/energy_response_mean": 1.0005868673324585, "metrics/particle/event/met_abs_error": 1.2088749408721924, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04363100603222847, "metrics/particle/event/scalar_pt_response_mean": 1.0010327100753784, "metrics/particle/event/vector_pt_closure": 0.04553806409239769, "metrics/particle/matched/delta_eta_abs_mean": 0.007036117371171713, "metrics/particle/matched/delta_phi_abs_mean": 0.006689951755106449, "metrics/particle/matched/delta_r_mean": 0.010862083174288273, "metrics/particle/matched/energy_relative_abs_mean": 0.05992727354168892, "metrics/particle/matched/pid_accuracy": 0.9820923805236816, "metrics/particle/matched/pt_relative_abs_mean": 0.06022225320339203, "metrics/particle/matching/duplicate_fraction": 0.014629947952926159, "metrics/particle/matching/efficiency": 0.9130808711051941, "metrics/particle/matching/f1": 0.9130808711051941, "metrics/particle/matching/purity": 0.9130808711051941, "metrics/particle/matching/target_coverage_dr0p05": 0.9018932580947876, "metrics/particle/matching/target_coverage_dr0p10": 0.9397590160369873, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.849397599697113, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9328743815422058}, "step_train_time": 0.4584386348724365, "step_valid_time": 2.8439248089998728, "step_total_time": 3189.78378534317, "elapsed_training_loop_seconds": 3189.78378534317, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0015107030454842, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.032769159503101, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9895470383275261, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.009302325581395349, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9837209302325581, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9825783972125436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.193055154999456, "step_plot_time": 3.55990250800005, "step_validation_cycle_time": 10.93347468399952} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_28000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_28000.json new file mode 100644 index 0000000000000000000000000000000000000000..a6cf28f1867f2b1e04f4b144031f4e479074d7b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_28000.json @@ -0,0 +1 @@ +{"epoch": 49.8220640569395, "train": {"Classification_binary": 0.056500405073165894, "Classification": 0.0002593866956885904, "Regression_pt": 0.025405965745449066, "Regression_eta": 9.929441876010969e-05, "Regression_sin_phi": 0.00019537577463779598, "Regression_cos_phi": 0.00018764386186376214, "Regression_energy": 0.024899814277887344, "Total": 0.8733537197113037, "Total_unweighted": 0.10754788584745256}, "train_group": {}, "valid": {"Classification_binary": 2.044257164001465, "Classification": 0.03162011504173279, "Regression_pt": 0.14149168133735657, "Regression_eta": 0.0008983648149296641, "Regression_sin_phi": 0.007912238128483295, "Regression_cos_phi": 0.0040129306726157665, "Regression_energy": 0.14110566675662994, "Total": 16.824222564697266, "Total_unweighted": 2.371298160753213, "metrics/particle/class_1/efficiency": 0.970251739025116, "metrics/particle/class_1/pid_accuracy": 0.9858490824699402, "metrics/particle/class_2/efficiency": 0.40740740299224854, "metrics/particle/class_2/pid_accuracy": 0.8181818127632141, "metrics/particle/class_3/efficiency": 0.8747474551200867, "metrics/particle/class_3/pid_accuracy": 0.986143171787262, "metrics/particle/class_4/efficiency": 0.9406779408454895, "metrics/particle/class_4/pid_accuracy": 0.954954981803894, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9882352948188782, "metrics/particle/count/bias_mean": -0.019999999552965164, "metrics/particle/count/mae": 0.2680000066757202, "metrics/particle/count/prediction_mean": 4.627999782562256, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.049551256000995636, "metrics/particle/event/energy_response_mean": 1.0059597492218018, "metrics/particle/event/met_abs_error": 1.2730085849761963, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04854746162891388, "metrics/particle/event/scalar_pt_response_mean": 1.0055090188980103, "metrics/particle/event/vector_pt_closure": 0.048810869455337524, "metrics/particle/matched/delta_eta_abs_mean": 0.0064057945273816586, "metrics/particle/matched/delta_phi_abs_mean": 0.008034116588532925, "metrics/particle/matched/delta_r_mean": 0.011342029087245464, "metrics/particle/matched/energy_relative_abs_mean": 0.05937102064490318, "metrics/particle/matched/pid_accuracy": 0.981203019618988, "metrics/particle/matched/pt_relative_abs_mean": 0.05967815965414047, "metrics/particle/matching/duplicate_fraction": 0.013828868046402931, "metrics/particle/matching/efficiency": 0.9156626462936401, "metrics/particle/matching/f1": 0.9176369309425354, "metrics/particle/matching/purity": 0.9196196794509888, "metrics/particle/matching/target_coverage_dr0p05": 0.9044750332832336, "metrics/particle/matching/target_coverage_dr0p10": 0.9423407912254333, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8450946807861328, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9328743815422058}, "step_train_time": 0.47070741653442383, "step_valid_time": 2.8438174940001772, "step_total_time": 3308.6415667533875, "elapsed_training_loop_seconds": 3308.6415667533875, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0028213897006093, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.031042773947676072, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 422, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9895470383275261, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.009302325581395349, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9791183294663574, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9813953488372092, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9802555168408827, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.226239514999179, "step_plot_time": 3.5227390820000437, "step_validation_cycle_time": 10.922411952999937} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_29000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_29000.json new file mode 100644 index 0000000000000000000000000000000000000000..e6b03276d40014473639a8bad903fd046a8ac896 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_29000.json @@ -0,0 +1 @@ +{"epoch": 51.60142348754449, "train": {"Classification_binary": 0.03713833540678024, "Classification": 2.7366343147150474e-06, "Regression_pt": 0.04249698668718338, "Regression_eta": 0.0002626795321702957, "Regression_sin_phi": 0.00010190004832111299, "Regression_cos_phi": 0.00012548181985039264, "Regression_energy": 0.04142386093735695, "Total": 1.0738495588302612, "Total_unweighted": 0.12155198106597709}, "train_group": {}, "valid": {"Classification_binary": 2.23492693901062, "Classification": 0.028034517541527748, "Regression_pt": 0.12564653158187866, "Regression_eta": 0.0011094370856881142, "Regression_sin_phi": 0.005989488679915667, "Regression_cos_phi": 0.0030584943015128374, "Regression_energy": 0.12622392177581787, "Total": 14.25130558013916, "Total_unweighted": 2.524989329976961, "metrics/particle/class_1/efficiency": 0.9771167039871216, "metrics/particle/class_1/pid_accuracy": 0.9882903695106506, "metrics/particle/class_2/efficiency": 0.5185185074806213, "metrics/particle/class_2/pid_accuracy": 0.7857142686843872, "metrics/particle/class_3/efficiency": 0.8787878751754761, "metrics/particle/class_3/pid_accuracy": 0.9908046126365662, "metrics/particle/class_4/efficiency": 0.9491525292396545, "metrics/particle/class_4/pid_accuracy": 0.9553571343421936, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9882352948188782, "metrics/particle/count/bias_mean": 0.004000000189989805, "metrics/particle/count/mae": 0.2680000066757202, "metrics/particle/count/prediction_mean": 4.6519999504089355, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04654872417449951, "metrics/particle/event/energy_response_mean": 0.9987314939498901, "metrics/particle/event/met_abs_error": 1.2535457611083984, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04563169181346893, "metrics/particle/event/scalar_pt_response_mean": 0.9981110692024231, "metrics/particle/event/vector_pt_closure": 0.046604935079813004, "metrics/particle/matched/delta_eta_abs_mean": 0.007059685420244932, "metrics/particle/matched/delta_phi_abs_mean": 0.006847083102911711, "metrics/particle/matched/delta_r_mean": 0.01089178305119276, "metrics/particle/matched/energy_relative_abs_mean": 0.0619693286716938, "metrics/particle/matched/pid_accuracy": 0.983224630355835, "metrics/particle/matched/pt_relative_abs_mean": 0.06223690137267113, "metrics/particle/matching/duplicate_fraction": 0.017196904867887497, "metrics/particle/matching/efficiency": 0.9234079122543335, "metrics/particle/matching/f1": 0.9230107665061951, "metrics/particle/matching/purity": 0.9226139187812805, "metrics/particle/matching/target_coverage_dr0p05": 0.9104991555213928, "metrics/particle/matching/target_coverage_dr0p10": 0.9457831382751465, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8519793748855591, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9363166689872742}, "step_train_time": 0.4563901424407959, "step_valid_time": 2.8234662060003757, "step_total_time": 3426.7538170814514, "elapsed_training_loop_seconds": 3426.7538170814514, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9974826469224866, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.029982008352591794, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9872241579558653, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.011627906976744186, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.986046511627907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9849012775842044, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9860788863109049, "step_test_time": 4.19827048299976, "step_plot_time": 3.5395427830007975, "step_validation_cycle_time": 10.890613770000527} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_3000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_3000.json new file mode 100644 index 0000000000000000000000000000000000000000..a18a62035b34175d92bd350098982d6305d1a6c1 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_3000.json @@ -0,0 +1 @@ +{"epoch": 5.338078291814947, "train": {"Classification_binary": 0.7159925103187561, "Classification": 0.002114551141858101, "Regression_pt": 0.05337180569767952, "Regression_eta": 0.0026158620603382587, "Regression_sin_phi": 0.0005377839552238584, "Regression_cos_phi": 0.0003024491888936609, "Regression_energy": 0.06225772574543953, "Total": 3.3016021251678467, "Total_unweighted": 0.837192688108189}, "train_group": {}, "valid": {"Classification_binary": 1.0627292394638062, "Classification": 0.009072283282876015, "Regression_pt": 0.09483602643013, "Regression_eta": 0.0010638827225193381, "Regression_sin_phi": 0.00932994857430458, "Regression_cos_phi": 0.004513965919613838, "Regression_energy": 0.09539090842008591, "Total": 14.60849666595459, "Total_unweighted": 1.2769362548133358, "metrics/particle/class_1/efficiency": 0.9794050455093384, "metrics/particle/class_1/pid_accuracy": 0.9906542301177979, "metrics/particle/class_2/efficiency": 0.5555555820465088, "metrics/particle/class_2/pid_accuracy": 0.800000011920929, "metrics/particle/class_3/efficiency": 0.8666666746139526, "metrics/particle/class_3/pid_accuracy": 0.9906759858131409, "metrics/particle/class_4/efficiency": 0.9406779408454895, "metrics/particle/class_4/pid_accuracy": 0.9729729890823364, "metrics/particle/class_5/efficiency": 0.9764705896377563, "metrics/particle/class_5/pid_accuracy": 0.9879518151283264, "metrics/particle/count/bias_mean": -0.02800000086426735, "metrics/particle/count/mae": 0.23600000143051147, "metrics/particle/count/prediction_mean": 4.619999885559082, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04261431843042374, "metrics/particle/event/energy_response_mean": 0.992497444152832, "metrics/particle/event/met_abs_error": 1.0851168632507324, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04207859933376312, "metrics/particle/event/scalar_pt_response_mean": 0.9931782484054565, "metrics/particle/event/vector_pt_closure": 0.04212789610028267, "metrics/particle/matched/delta_eta_abs_mean": 0.00751615222543478, "metrics/particle/matched/delta_phi_abs_mean": 0.00675912294536829, "metrics/particle/matched/delta_r_mean": 0.011281772516667843, "metrics/particle/matched/energy_relative_abs_mean": 0.060787808150053024, "metrics/particle/matched/pid_accuracy": 0.9859287142753601, "metrics/particle/matched/pt_relative_abs_mean": 0.061405252665281296, "metrics/particle/matching/duplicate_fraction": 0.017316017299890518, "metrics/particle/matching/efficiency": 0.9173838496208191, "metrics/particle/matching/f1": 0.9201553463935852, "metrics/particle/matching/purity": 0.9229437112808228, "metrics/particle/matching/target_coverage_dr0p05": 0.9061962366104126, "metrics/particle/matching/target_coverage_dr0p10": 0.9457831382751465, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8459552526473999, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9363166689872742}, "step_train_time": 0.4431600570678711, "step_valid_time": 2.837288520000129, "step_total_time": 347.23032450675964, "elapsed_training_loop_seconds": 347.23032450675964, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9970716705827208, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.029985287023460705, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.986046511627907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9849012775842044, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.013953488372093023, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9837209302325581, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9825783972125436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9837587006960556, "step_test_time": 4.143955807000111, "step_plot_time": 3.4900571480002327, "step_validation_cycle_time": 10.784292693000225} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..f36e5d6f3926a52bbea336f5991bd0c5ac31fe4f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 0.04905742034316063, "Classification": 1.1042805425631741e-07, "Regression_pt": 0.012888423167169094, "Regression_eta": 0.00016762828454375267, "Regression_sin_phi": 4.4358701416058466e-05, "Regression_cos_phi": 6.124458741396666e-05, "Regression_energy": 0.012541101314127445, "Total": 0.41303902864456177, "Total_unweighted": 0.0747602868258852}, "train_group": {}, "valid": {"Classification_binary": 2.378368854522705, "Classification": 0.030331846326589584, "Regression_pt": 0.12847214937210083, "Regression_eta": 0.0010021375492215157, "Regression_sin_phi": 0.009968108497560024, "Regression_cos_phi": 0.004681763704866171, "Regression_energy": 0.12817467749118805, "Total": 18.95298957824707, "Total_unweighted": 2.6809995374642313, "metrics/particle/class_1/efficiency": 0.9816933870315552, "metrics/particle/class_1/pid_accuracy": 0.9860140085220337, "metrics/particle/class_2/efficiency": 0.40740740299224854, "metrics/particle/class_2/pid_accuracy": 0.9090909361839294, "metrics/particle/class_3/efficiency": 0.868686854839325, "metrics/particle/class_3/pid_accuracy": 0.9906976819038391, "metrics/particle/class_4/efficiency": 0.9491525292396545, "metrics/particle/class_4/pid_accuracy": 0.9642857313156128, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 0.988095223903656, "metrics/particle/count/bias_mean": 0.0, "metrics/particle/count/mae": 0.2720000147819519, "metrics/particle/count/prediction_mean": 4.6479997634887695, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.049470629543066025, "metrics/particle/event/energy_response_mean": 1.003188133239746, "metrics/particle/event/met_abs_error": 1.2729763984680176, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04834606125950813, "metrics/particle/event/scalar_pt_response_mean": 1.0022521018981934, "metrics/particle/event/vector_pt_closure": 0.04971444234251976, "metrics/particle/matched/delta_eta_abs_mean": 0.007238189224153757, "metrics/particle/matched/delta_phi_abs_mean": 0.0069501386024057865, "metrics/particle/matched/delta_r_mean": 0.011095864698290825, "metrics/particle/matched/energy_relative_abs_mean": 0.0630209669470787, "metrics/particle/matched/pid_accuracy": 0.9849905967712402, "metrics/particle/matched/pt_relative_abs_mean": 0.0634477511048317, "metrics/particle/matching/duplicate_fraction": 0.013769363053143024, "metrics/particle/matching/efficiency": 0.9173838496208191, "metrics/particle/matching/f1": 0.9173838496208191, "metrics/particle/matching/purity": 0.9173838496208191, "metrics/particle/matching/target_coverage_dr0p05": 0.9010326862335205, "metrics/particle/matching/target_coverage_dr0p10": 0.9414802193641663, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.849397599697113, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9363166689872742}, "step_train_time": 0.49791693687438965, "step_valid_time": 2.830745520999699, "step_total_time": 3545.2912487983704, "elapsed_training_loop_seconds": 3545.2912487983704, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0033817605098965, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.030471800004538085, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9929742388758782, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9883449883449883, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.00702576112412178, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9929742388758782, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9883449883449883, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9837587006960556, "step_test_time": 4.195509135000066, "step_plot_time": 3.5380867689991646, "step_validation_cycle_time": 10.89262729100028} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_31000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_31000.json new file mode 100644 index 0000000000000000000000000000000000000000..a5adf5c0dbb6830ad120c98cd7d5244a1ac70b9b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_31000.json @@ -0,0 +1 @@ +{"epoch": 55.16014234875445, "train": {"Classification_binary": 0.10900285840034485, "Classification": 1.3771139037999092e-06, "Regression_pt": 0.023748962208628654, "Regression_eta": 0.0001336872373940423, "Regression_sin_phi": 8.60495274537243e-05, "Regression_cos_phi": 5.250278263702057e-05, "Regression_energy": 0.023692656308412552, "Total": 0.7005030512809753, "Total_unweighted": 0.15671809357877464}, "train_group": {}, "valid": {"Classification_binary": 2.3217380046844482, "Classification": 0.027572665363550186, "Regression_pt": 0.12736158072948456, "Regression_eta": 0.0011325865052640438, "Regression_sin_phi": 0.010540884919464588, "Regression_cos_phi": 0.005086318124085665, "Regression_energy": 0.12707403302192688, "Total": 19.42679786682129, "Total_unweighted": 2.620506073348224, "metrics/particle/class_1/efficiency": 0.9656750559806824, "metrics/particle/class_1/pid_accuracy": 0.9834123253822327, "metrics/particle/class_2/efficiency": 0.5555555820465088, "metrics/particle/class_2/pid_accuracy": 0.7333333492279053, "metrics/particle/class_3/efficiency": 0.8868687152862549, "metrics/particle/class_3/pid_accuracy": 0.9863325953483582, "metrics/particle/class_4/efficiency": 0.9322034120559692, "metrics/particle/class_4/pid_accuracy": 0.9545454382896423, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 0.988095223903656, "metrics/particle/count/bias_mean": -0.024000000208616257, "metrics/particle/count/mae": 0.2639999985694885, "metrics/particle/count/prediction_mean": 4.624000072479248, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.047927096486091614, "metrics/particle/event/energy_response_mean": 0.9993391036987305, "metrics/particle/event/met_abs_error": 1.197920560836792, "metrics/particle/event/scalar_pt_relative_abs_error": 0.046079594641923904, "metrics/particle/event/scalar_pt_response_mean": 0.9979416728019714, "metrics/particle/event/vector_pt_closure": 0.04592881351709366, "metrics/particle/matched/delta_eta_abs_mean": 0.007249635644257069, "metrics/particle/matched/delta_phi_abs_mean": 0.006905015558004379, "metrics/particle/matched/delta_r_mean": 0.011203806847333908, "metrics/particle/matched/energy_relative_abs_mean": 0.06104588881134987, "metrics/particle/matched/pid_accuracy": 0.9785046577453613, "metrics/particle/matched/pt_relative_abs_mean": 0.06158745288848877, "metrics/particle/matching/duplicate_fraction": 0.014705882407724857, "metrics/particle/matching/efficiency": 0.9208261370658875, "metrics/particle/matching/f1": 0.9232096672058105, "metrics/particle/matching/purity": 0.9256055355072021, "metrics/particle/matching/target_coverage_dr0p05": 0.9044750332832336, "metrics/particle/matching/target_coverage_dr0p10": 0.9440619349479675, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.849397599697113, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9337349534034729}, "step_train_time": 0.48273539543151855, "step_valid_time": 2.891199151999899, "step_total_time": 3664.00674366951, "elapsed_training_loop_seconds": 3664.00674366951, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0046325048698719, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.03315347024416426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.006993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9883449883449883, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.986046511627907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.229501585999969, "step_plot_time": 3.5465118669999356, "step_validation_cycle_time": 10.99924803200065} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_32000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_32000.json new file mode 100644 index 0000000000000000000000000000000000000000..154057058f2e59bf9cc3ee7ea79a25149dd317b9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_32000.json @@ -0,0 +1 @@ +{"epoch": 56.93950177935943, "train": {"Classification_binary": 0.09996071457862854, "Classification": 2.3737693481962197e-06, "Regression_pt": 0.024435358121991158, "Regression_eta": 0.0006345638539642096, "Regression_sin_phi": 0.00011642275785561651, "Regression_cos_phi": 0.00012593955034390092, "Regression_energy": 0.02727004699409008, "Total": 0.9360268115997314, "Total_unweighted": 0.1525454196262217}, "train_group": {}, "valid": {"Classification_binary": 2.468918800354004, "Classification": 0.02967608906328678, "Regression_pt": 0.11993418633937836, "Regression_eta": 0.0009871214861050248, "Regression_sin_phi": 0.010582124814391136, "Regression_cos_phi": 0.005301938392221928, "Regression_energy": 0.1212729960680008, "Total": 19.77685546875, "Total_unweighted": 2.756673256517388, "metrics/particle/class_1/efficiency": 0.9771167039871216, "metrics/particle/class_1/pid_accuracy": 0.9882903695106506, "metrics/particle/class_2/efficiency": 0.48148149251937866, "metrics/particle/class_2/pid_accuracy": 0.8461538553237915, "metrics/particle/class_3/efficiency": 0.8727272748947144, "metrics/particle/class_3/pid_accuracy": 0.9861111044883728, "metrics/particle/class_4/efficiency": 0.9491525292396545, "metrics/particle/class_4/pid_accuracy": 0.9285714030265808, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9882352948188782, "metrics/particle/count/bias_mean": -0.004000000189989805, "metrics/particle/count/mae": 0.2840000092983246, "metrics/particle/count/prediction_mean": 4.644000053405762, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04813963174819946, "metrics/particle/event/energy_response_mean": 1.0000137090682983, "metrics/particle/event/met_abs_error": 1.2630163431167603, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04772408306598663, "metrics/particle/event/scalar_pt_response_mean": 1.0003905296325684, "metrics/particle/event/vector_pt_closure": 0.047321777790784836, "metrics/particle/matched/delta_eta_abs_mean": 0.006645347457379103, "metrics/particle/matched/delta_phi_abs_mean": 0.0067237685434520245, "metrics/particle/matched/delta_r_mean": 0.010514041408896446, "metrics/particle/matched/energy_relative_abs_mean": 0.06109098717570305, "metrics/particle/matched/pid_accuracy": 0.979420006275177, "metrics/particle/matched/pt_relative_abs_mean": 0.06143394857645035, "metrics/particle/matching/duplicate_fraction": 0.013781223446130753, "metrics/particle/matching/efficiency": 0.9199655652046204, "metrics/particle/matching/f1": 0.9203615784645081, "metrics/particle/matching/purity": 0.9207579493522644, "metrics/particle/matching/target_coverage_dr0p05": 0.9053356051445007, "metrics/particle/matching/target_coverage_dr0p10": 0.9423407912254333, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8433734774589539, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9380378723144531}, "step_train_time": 0.4737362861633301, "step_valid_time": 2.837063971000134, "step_total_time": 3783.0443608760834, "elapsed_training_loop_seconds": 3783.0443608760834, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0048741301068413, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.027611594006220597, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 428, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9929906542056075, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.989522700814901, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.007009345794392523, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9883177570093458, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9848661233993015, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9860788863109049, "step_test_time": 4.212237036000261, "step_plot_time": 3.5775402310000572, "step_validation_cycle_time": 10.960316411000349} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_33000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_33000.json new file mode 100644 index 0000000000000000000000000000000000000000..e093d678668e375f44b0fcbf764be6d2707dce5f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_33000.json @@ -0,0 +1 @@ +{"epoch": 58.718861209964416, "train": {"Classification_binary": 0.05268939211964607, "Classification": 8.858069122652523e-06, "Regression_pt": 0.014688017778098583, "Regression_eta": 0.0002736345923040062, "Regression_sin_phi": 0.00011167665797984228, "Regression_cos_phi": 0.00022995956533122808, "Regression_energy": 0.014570478349924088, "Total": 0.6566556692123413, "Total_unweighted": 0.08257201713240647}, "train_group": {}, "valid": {"Classification_binary": 2.5910425186157227, "Classification": 0.02983175776898861, "Regression_pt": 0.13647735118865967, "Regression_eta": 0.000990396598353982, "Regression_sin_phi": 0.007386083249002695, "Regression_cos_phi": 0.003484446322545409, "Regression_energy": 0.13773205876350403, "Total": 16.36179542541504, "Total_unweighted": 2.906944612506777, "metrics/particle/class_1/efficiency": 0.972540020942688, "metrics/particle/class_1/pid_accuracy": 0.9835293889045715, "metrics/particle/class_2/efficiency": 0.5185185074806213, "metrics/particle/class_2/pid_accuracy": 0.7857142686843872, "metrics/particle/class_3/efficiency": 0.8848484754562378, "metrics/particle/class_3/pid_accuracy": 0.9885844588279724, "metrics/particle/class_4/efficiency": 0.9322034120559692, "metrics/particle/class_4/pid_accuracy": 0.9545454382896423, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9764705896377563, "metrics/particle/count/bias_mean": -0.01600000075995922, "metrics/particle/count/mae": 0.25600001215934753, "metrics/particle/count/prediction_mean": 4.631999969482422, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04828132316470146, "metrics/particle/event/energy_response_mean": 0.990633487701416, "metrics/particle/event/met_abs_error": 1.2410502433776855, "metrics/particle/event/scalar_pt_relative_abs_error": 0.048096850514411926, "metrics/particle/event/scalar_pt_response_mean": 0.9913315773010254, "metrics/particle/event/vector_pt_closure": 0.04883702099323273, "metrics/particle/matched/delta_eta_abs_mean": 0.007879175245761871, "metrics/particle/matched/delta_phi_abs_mean": 0.006941832136362791, "metrics/particle/matched/delta_r_mean": 0.011648958548903465, "metrics/particle/matched/energy_relative_abs_mean": 0.061600957065820694, "metrics/particle/matched/pid_accuracy": 0.97947758436203, "metrics/particle/matched/pt_relative_abs_mean": 0.06174926459789276, "metrics/particle/matching/duplicate_fraction": 0.012089810334146023, "metrics/particle/matching/efficiency": 0.9225473403930664, "metrics/particle/matching/f1": 0.9241379499435425, "metrics/particle/matching/purity": 0.9257340431213379, "metrics/particle/matching/target_coverage_dr0p05": 0.9027538895606995, "metrics/particle/matching/target_coverage_dr0p10": 0.9423407912254333, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.848537027835846, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9397590160369873}, "step_train_time": 0.49060916900634766, "step_valid_time": 2.8378902569993443, "step_total_time": 3901.9434158802032, "elapsed_training_loop_seconds": 3901.9434158802032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9973092836023636, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.030154322450796855, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.006993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9906759906759907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.210180579000735, "step_plot_time": 3.556768569000269, "step_validation_cycle_time": 10.945237445000203} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_34000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_34000.json new file mode 100644 index 0000000000000000000000000000000000000000..36dbb4727d67aaf4950766026adaf20fa712f957 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_34000.json @@ -0,0 +1 @@ +{"epoch": 60.49822064056939, "train": {"Classification_binary": 0.294736385345459, "Classification": 1.5547767588941497e-06, "Regression_pt": 0.03610679879784584, "Regression_eta": 0.00018157466547563672, "Regression_sin_phi": 0.00010671452764654532, "Regression_cos_phi": 3.205017856089398e-05, "Regression_energy": 0.03598743677139282, "Total": 1.1343032121658325, "Total_unweighted": 0.3671525150631396}, "train_group": {}, "valid": {"Classification_binary": 2.549992322921753, "Classification": 0.032068002969026566, "Regression_pt": 0.1300608366727829, "Regression_eta": 0.0010413635754957795, "Regression_sin_phi": 0.005800668615847826, "Regression_cos_phi": 0.0031374881509691477, "Regression_energy": 0.12980560958385468, "Total": 14.870417594909668, "Total_unweighted": 2.85190629248973, "metrics/particle/class_1/efficiency": 0.9679633975028992, "metrics/particle/class_1/pid_accuracy": 0.9905437231063843, "metrics/particle/class_2/efficiency": 0.5555555820465088, "metrics/particle/class_2/pid_accuracy": 0.7333333492279053, "metrics/particle/class_3/efficiency": 0.8828282952308655, "metrics/particle/class_3/pid_accuracy": 0.9885583519935608, "metrics/particle/class_4/efficiency": 0.9406779408454895, "metrics/particle/class_4/pid_accuracy": 0.954954981803894, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9764705896377563, "metrics/particle/count/bias_mean": 0.00800000037997961, "metrics/particle/count/mae": 0.23199999332427979, "metrics/particle/count/prediction_mean": 4.656000137329102, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.045712005347013474, "metrics/particle/event/energy_response_mean": 0.9924269318580627, "metrics/particle/event/met_abs_error": 1.2705707550048828, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04533408209681511, "metrics/particle/event/scalar_pt_response_mean": 0.9909427165985107, "metrics/particle/event/vector_pt_closure": 0.04564320296049118, "metrics/particle/matched/delta_eta_abs_mean": 0.007021729834377766, "metrics/particle/matched/delta_phi_abs_mean": 0.007062074262648821, "metrics/particle/matched/delta_r_mean": 0.011105060577392578, "metrics/particle/matched/energy_relative_abs_mean": 0.060723938047885895, "metrics/particle/matched/pid_accuracy": 0.9813258647918701, "metrics/particle/matched/pt_relative_abs_mean": 0.06115114316344261, "metrics/particle/matching/duplicate_fraction": 0.015463917516171932, "metrics/particle/matching/efficiency": 0.9216867685317993, "metrics/particle/matching/f1": 0.9208942651748657, "metrics/particle/matching/purity": 0.9201030731201172, "metrics/particle/matching/target_coverage_dr0p05": 0.9104991555213928, "metrics/particle/matching/target_coverage_dr0p10": 0.9492254853248596, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.846815824508667, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9388984441757202}, "step_train_time": 0.4770629405975342, "step_valid_time": 2.886664163000205, "step_total_time": 4020.5232393741608, "elapsed_training_loop_seconds": 4020.5232393741608, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0011838405106739, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02794928605574276, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 428, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9929906542056075, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.989522700814901, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.007009345794392523, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9883177570093458, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9848661233993015, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9860788863109049, "step_test_time": 4.2850580879994595, "step_plot_time": 3.5407291459996486, "step_validation_cycle_time": 11.04893433899997} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_35000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_35000.json new file mode 100644 index 0000000000000000000000000000000000000000..97109f0cc99ca3651d42f2825dcea4a9762870b9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_35000.json @@ -0,0 +1 @@ +{"epoch": 62.27758007117438, "train": {"Classification_binary": 0.13905969262123108, "Classification": 2.5632200504333014e-06, "Regression_pt": 0.009866512380540371, "Regression_eta": 0.0002927200694102794, "Regression_sin_phi": 6.0752019635401666e-05, "Regression_cos_phi": 4.327279748395085e-05, "Regression_energy": 0.010541548952460289, "Total": 0.48708969354629517, "Total_unweighted": 0.1598670620608118}, "train_group": {}, "valid": {"Classification_binary": 2.8094594478607178, "Classification": 0.0326034277677536, "Regression_pt": 0.12284105271100998, "Regression_eta": 0.0009260731167159975, "Regression_sin_phi": 0.0100339250639081, "Regression_cos_phi": 0.00431338744238019, "Regression_energy": 0.12289518862962723, "Total": 19.23674964904785, "Total_unweighted": 3.103072502592113, "metrics/particle/class_1/efficiency": 0.9679633975028992, "metrics/particle/class_1/pid_accuracy": 0.9810874462127686, "metrics/particle/class_2/efficiency": 0.5185185074806213, "metrics/particle/class_2/pid_accuracy": 0.8571428656578064, "metrics/particle/class_3/efficiency": 0.8868687152862549, "metrics/particle/class_3/pid_accuracy": 0.9931662678718567, "metrics/particle/class_4/efficiency": 0.9237288236618042, "metrics/particle/class_4/pid_accuracy": 0.9541284441947937, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9882352948188782, "metrics/particle/count/bias_mean": -0.01600000075995922, "metrics/particle/count/mae": 0.29600000381469727, "metrics/particle/count/prediction_mean": 4.631999969482422, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04683861881494522, "metrics/particle/event/energy_response_mean": 0.9941757321357727, "metrics/particle/event/met_abs_error": 1.2116222381591797, "metrics/particle/event/scalar_pt_relative_abs_error": 0.046748511493206024, "metrics/particle/event/scalar_pt_response_mean": 0.9919381737709045, "metrics/particle/event/vector_pt_closure": 0.04695306718349457, "metrics/particle/matched/delta_eta_abs_mean": 0.0073298197239637375, "metrics/particle/matched/delta_phi_abs_mean": 0.007275676820427179, "metrics/particle/matched/delta_r_mean": 0.011503883637487888, "metrics/particle/matched/energy_relative_abs_mean": 0.060101818293333054, "metrics/particle/matched/pid_accuracy": 0.982243001461029, "metrics/particle/matched/pt_relative_abs_mean": 0.06061554700136185, "metrics/particle/matching/duplicate_fraction": 0.011226251721382141, "metrics/particle/matching/efficiency": 0.9208261370658875, "metrics/particle/matching/f1": 0.9224137663841248, "metrics/particle/matching/purity": 0.9240068793296814, "metrics/particle/matching/target_coverage_dr0p05": 0.9070568084716797, "metrics/particle/matching/target_coverage_dr0p10": 0.9414802193641663, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8450946807861328, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9311531782150269}, "step_train_time": 0.46439385414123535, "step_valid_time": 2.8718405520003216, "step_total_time": 4139.673734903336, "elapsed_training_loop_seconds": 4139.673734903336, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0002322764910272, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02866781928601525, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 428, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9929906542056075, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.989522700814901, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.007009345794392523, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9929906542056075, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.989522700814901, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9860788863109049, "step_test_time": 4.22563880799953, "step_plot_time": 3.5380266830006804, "step_validation_cycle_time": 10.971906142999615} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_4000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_4000.json new file mode 100644 index 0000000000000000000000000000000000000000..4871216b9e30df45f056943bbc6f96ad385f10a9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_4000.json @@ -0,0 +1 @@ +{"epoch": 7.117437722419929, "train": {"Classification_binary": 1.6029752492904663, "Classification": 0.0035004725214093924, "Regression_pt": 0.11968192458152771, "Regression_eta": 0.00041688853525556624, "Regression_sin_phi": 0.00041382783092558384, "Regression_cos_phi": 0.0003318679518997669, "Regression_energy": 0.12177854031324387, "Total": 4.867015361785889, "Total_unweighted": 1.8490987710247282}, "train_group": {}, "valid": {"Classification_binary": 1.0533006191253662, "Classification": 0.008877480402588844, "Regression_pt": 0.08769708126783371, "Regression_eta": 0.001071634702384472, "Regression_sin_phi": 0.006175502203404903, "Regression_cos_phi": 0.0028703443240374327, "Regression_energy": 0.08766964077949524, "Total": 10.73813533782959, "Total_unweighted": 1.2476623028051108, "metrics/particle/class_1/efficiency": 0.9794050455093384, "metrics/particle/class_1/pid_accuracy": 0.9836448431015015, "metrics/particle/class_2/efficiency": 0.5185185074806213, "metrics/particle/class_2/pid_accuracy": 0.9285714030265808, "metrics/particle/class_3/efficiency": 0.8666666746139526, "metrics/particle/class_3/pid_accuracy": 0.9930070042610168, "metrics/particle/class_4/efficiency": 0.9406779408454895, "metrics/particle/class_4/pid_accuracy": 0.954954981803894, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 0.988095223903656, "metrics/particle/count/bias_mean": -0.035999998450279236, "metrics/particle/count/mae": 0.23600000143051147, "metrics/particle/count/prediction_mean": 4.611999988555908, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.044349219650030136, "metrics/particle/event/energy_response_mean": 0.9981241822242737, "metrics/particle/event/met_abs_error": 1.0598540306091309, "metrics/particle/event/scalar_pt_relative_abs_error": 0.044437799602746964, "metrics/particle/event/scalar_pt_response_mean": 0.9988625049591064, "metrics/particle/event/vector_pt_closure": 0.04087162762880325, "metrics/particle/matched/delta_eta_abs_mean": 0.007727058604359627, "metrics/particle/matched/delta_phi_abs_mean": 0.0067122760228812695, "metrics/particle/matched/delta_r_mean": 0.011354444548487663, "metrics/particle/matched/energy_relative_abs_mean": 0.06263139843940735, "metrics/particle/matched/pid_accuracy": 0.9840525388717651, "metrics/particle/matched/pt_relative_abs_mean": 0.06341997534036636, "metrics/particle/matching/duplicate_fraction": 0.014744145795702934, "metrics/particle/matching/efficiency": 0.9173838496208191, "metrics/particle/matching/f1": 0.9209503531455994, "metrics/particle/matching/purity": 0.9245446920394897, "metrics/particle/matching/target_coverage_dr0p05": 0.9044750332832336, "metrics/particle/matching/target_coverage_dr0p10": 0.9449225664138794, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8476763963699341, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9354560971260071}, "step_train_time": 0.44715332984924316, "step_valid_time": 2.8155480440000247, "step_total_time": 465.45292234420776, "elapsed_training_loop_seconds": 465.45292234420776, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0010269381877128, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.029831871585116687, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 432, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9837962962962963, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.984936268829664, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.016203703703703703, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9814814814814815, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9826187717265353, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9860788863109049, "step_test_time": 4.124640055000327, "step_plot_time": 3.5159118830001717, "step_validation_cycle_time": 10.783847999000045} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_5000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_5000.json new file mode 100644 index 0000000000000000000000000000000000000000..e161251cd70122ad4d7ac889f1f7f7cf1b58d3aa --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_5000.json @@ -0,0 +1 @@ +{"epoch": 8.896797153024911, "train": {"Classification_binary": 0.8234134316444397, "Classification": 0.002962153870612383, "Regression_pt": 0.024485863745212555, "Regression_eta": 0.0005607950151897967, "Regression_sin_phi": 0.00016172272444237024, "Regression_cos_phi": 0.0005957891698926687, "Regression_energy": 0.027631167322397232, "Total": 2.269857883453369, "Total_unweighted": 0.8798109234921867}, "train_group": {}, "valid": {"Classification_binary": 1.0457738637924194, "Classification": 0.008590911515057087, "Regression_pt": 0.0911758542060852, "Regression_eta": 0.0010197041556239128, "Regression_sin_phi": 0.0037228246219456196, "Regression_cos_phi": 0.0018086822237819433, "Regression_energy": 0.09134788811206818, "Total": 8.03724479675293, "Total_unweighted": 1.2434397286269814, "metrics/particle/class_1/efficiency": 0.9794050455093384, "metrics/particle/class_1/pid_accuracy": 0.9813084006309509, "metrics/particle/class_2/efficiency": 0.5185185074806213, "metrics/particle/class_2/pid_accuracy": 0.8571428656578064, "metrics/particle/class_3/efficiency": 0.870707094669342, "metrics/particle/class_3/pid_accuracy": 0.9953595995903015, "metrics/particle/class_4/efficiency": 0.9322034120559692, "metrics/particle/class_4/pid_accuracy": 0.9545454382896423, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": 0.0, "metrics/particle/count/mae": 0.23999999463558197, "metrics/particle/count/prediction_mean": 4.6479997634887695, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.043038010597229004, "metrics/particle/event/energy_response_mean": 0.9949750304222107, "metrics/particle/event/met_abs_error": 1.0576039552688599, "metrics/particle/event/scalar_pt_relative_abs_error": 0.042957086116075516, "metrics/particle/event/scalar_pt_response_mean": 0.9941492080688477, "metrics/particle/event/vector_pt_closure": 0.04359503835439682, "metrics/particle/matched/delta_eta_abs_mean": 0.007448645308613777, "metrics/particle/matched/delta_phi_abs_mean": 0.006959159858524799, "metrics/particle/matched/delta_r_mean": 0.01130000501871109, "metrics/particle/matched/energy_relative_abs_mean": 0.06267502903938293, "metrics/particle/matched/pid_accuracy": 0.9840674996376038, "metrics/particle/matched/pt_relative_abs_mean": 0.06327742338180542, "metrics/particle/matching/duplicate_fraction": 0.014629947952926159, "metrics/particle/matching/efficiency": 0.9182444214820862, "metrics/particle/matching/f1": 0.9182444214820862, "metrics/particle/matching/purity": 0.9182444214820862, "metrics/particle/matching/target_coverage_dr0p05": 0.9053356051445007, "metrics/particle/matching/target_coverage_dr0p10": 0.9466437101364136, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8425129055976868, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9354560971260071}, "step_train_time": 0.4469337463378906, "step_valid_time": 2.8662541359999523, "step_total_time": 583.0520930290222, "elapsed_training_loop_seconds": 583.0520930290222, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9974965063119308, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.03189044538031405, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.013921113689095127, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9860788863109049, "step_test_time": 4.164072731000033, "step_plot_time": 3.508592798000336, "step_validation_cycle_time": 10.925750072000028} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_6000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_6000.json new file mode 100644 index 0000000000000000000000000000000000000000..72c8394cdb36738d36d4794d00a71bc3b7572c1b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_6000.json @@ -0,0 +1 @@ +{"epoch": 10.676156583629894, "train": {"Classification_binary": 0.8981099724769592, "Classification": 0.0008416433702223003, "Regression_pt": 0.060282789170742035, "Regression_eta": 0.0009886606130748987, "Regression_sin_phi": 0.00019650977628771216, "Regression_cos_phi": 0.0001985496492125094, "Regression_energy": 0.03873584419488907, "Total": 2.4641926288604736, "Total_unweighted": 0.9993539692513878}, "train_group": {}, "valid": {"Classification_binary": 1.0893213748931885, "Classification": 0.009079026989638805, "Regression_pt": 0.09449154883623123, "Regression_eta": 0.00110638199839741, "Regression_sin_phi": 0.006188176106661558, "Regression_cos_phi": 0.002755001187324524, "Regression_energy": 0.09426874667406082, "Total": 10.853589057922363, "Total_unweighted": 1.2972102566855028, "metrics/particle/class_1/efficiency": 0.9794050455093384, "metrics/particle/class_1/pid_accuracy": 0.985981285572052, "metrics/particle/class_2/efficiency": 0.48148149251937866, "metrics/particle/class_2/pid_accuracy": 0.8461538553237915, "metrics/particle/class_3/efficiency": 0.8646464943885803, "metrics/particle/class_3/pid_accuracy": 0.9929906725883484, "metrics/particle/class_4/efficiency": 0.9322034120559692, "metrics/particle/class_4/pid_accuracy": 0.9545454382896423, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.019999999552965164, "metrics/particle/count/mae": 0.23600000143051147, "metrics/particle/count/prediction_mean": 4.627999782562256, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.041127316653728485, "metrics/particle/event/energy_response_mean": 0.9979801774024963, "metrics/particle/event/met_abs_error": 1.0517436265945435, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04220978915691376, "metrics/particle/event/scalar_pt_response_mean": 0.9959724545478821, "metrics/particle/event/vector_pt_closure": 0.04016345366835594, "metrics/particle/matched/delta_eta_abs_mean": 0.007567484863102436, "metrics/particle/matched/delta_phi_abs_mean": 0.006246260832995176, "metrics/particle/matched/delta_r_mean": 0.01096530631184578, "metrics/particle/matched/energy_relative_abs_mean": 0.06134679913520813, "metrics/particle/matched/pid_accuracy": 0.9849482774734497, "metrics/particle/matched/pt_relative_abs_mean": 0.0618228055536747, "metrics/particle/matching/duplicate_fraction": 0.016421779990196228, "metrics/particle/matching/efficiency": 0.914802074432373, "metrics/particle/matching/f1": 0.9167744517326355, "metrics/particle/matching/purity": 0.9187554121017456, "metrics/particle/matching/target_coverage_dr0p05": 0.9061962366104126, "metrics/particle/matching/target_coverage_dr0p10": 0.9440619349479675, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.846815824508667, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.932013750076294}, "step_train_time": 0.444730281829834, "step_valid_time": 2.8177511810004034, "step_total_time": 701.3697824478149, "elapsed_training_loop_seconds": 701.3697824478149, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9973064665253113, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02790707321752972, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 432, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9814814814814815, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9826187717265353, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.018518518518518517, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9791666666666666, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9803012746234068, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9837587006960556, "step_test_time": 4.141498895000041, "step_plot_time": 3.523761843000102, "step_validation_cycle_time": 10.792690539999967} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_7000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_7000.json new file mode 100644 index 0000000000000000000000000000000000000000..339779c92d471cabaa576b1bd3294efb6e8c6c92 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_7000.json @@ -0,0 +1 @@ +{"epoch": 12.455516014234876, "train": {"Classification_binary": 0.8994888663291931, "Classification": 0.00021222479699645191, "Regression_pt": 0.04126988723874092, "Regression_eta": 0.0003775709483306855, "Regression_sin_phi": 0.00030699072522111237, "Regression_cos_phi": 0.0001627063611522317, "Regression_energy": 0.03929613530635834, "Total": 2.141533374786377, "Total_unweighted": 0.9811143817059929}, "train_group": {}, "valid": {"Classification_binary": 1.0643616914749146, "Classification": 0.009679381735622883, "Regression_pt": 0.09316781163215637, "Regression_eta": 0.0010560950031504035, "Regression_sin_phi": 0.006893044803291559, "Regression_cos_phi": 0.0034661265090107918, "Regression_energy": 0.093232162296772, "Total": 11.921224594116211, "Total_unweighted": 1.2718563134549186, "metrics/particle/class_1/efficiency": 0.9748283624649048, "metrics/particle/class_1/pid_accuracy": 0.98591548204422, "metrics/particle/class_2/efficiency": 0.5555555820465088, "metrics/particle/class_2/pid_accuracy": 0.8666666746139526, "metrics/particle/class_3/efficiency": 0.8808080554008484, "metrics/particle/class_3/pid_accuracy": 0.9931192398071289, "metrics/particle/class_4/efficiency": 0.9406779408454895, "metrics/particle/class_4/pid_accuracy": 0.954954981803894, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": 0.012000000104308128, "metrics/particle/count/mae": 0.2199999988079071, "metrics/particle/count/prediction_mean": 4.659999847412109, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04154738038778305, "metrics/particle/event/energy_response_mean": 1.0013375282287598, "metrics/particle/event/met_abs_error": 1.038639783859253, "metrics/particle/event/scalar_pt_relative_abs_error": 0.041913557797670364, "metrics/particle/event/scalar_pt_response_mean": 1.0004570484161377, "metrics/particle/event/vector_pt_closure": 0.04103200510144234, "metrics/particle/matched/delta_eta_abs_mean": 0.007219352293759584, "metrics/particle/matched/delta_phi_abs_mean": 0.006635793950408697, "metrics/particle/matched/delta_r_mean": 0.010829840786755085, "metrics/particle/matched/energy_relative_abs_mean": 0.0619179792702198, "metrics/particle/matched/pid_accuracy": 0.9850746393203735, "metrics/particle/matched/pt_relative_abs_mean": 0.062210675328969955, "metrics/particle/matching/duplicate_fraction": 0.014592275023460388, "metrics/particle/matching/efficiency": 0.9225473403930664, "metrics/particle/matching/f1": 0.9213579893112183, "metrics/particle/matching/purity": 0.9201716780662537, "metrics/particle/matching/target_coverage_dr0p05": 0.9087779521942139, "metrics/particle/matching/target_coverage_dr0p10": 0.9492254853248596, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8537005186080933, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9414802193641663}, "step_train_time": 0.45763540267944336, "step_valid_time": 2.79438727500019, "step_total_time": 818.9208035469055, "elapsed_training_loop_seconds": 818.9208035469055, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9974174630513126, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.029614845564968406, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 432, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9861111111111112, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9872537659327926, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.013888888888888888, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9837962962962963, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.984936268829664, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.190939383000114, "step_plot_time": 3.4813453100000515, "step_validation_cycle_time": 10.781911289000163} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_8000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_8000.json new file mode 100644 index 0000000000000000000000000000000000000000..a20691e0f7d58c7e5ada7f7b792f21968a45c927 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_8000.json @@ -0,0 +1 @@ +{"epoch": 14.234875444839858, "train": {"Classification_binary": 0.5270521640777588, "Classification": 0.002084465464577079, "Regression_pt": 0.04247331619262695, "Regression_eta": 0.0009419736452400684, "Regression_sin_phi": 0.00014964447473175824, "Regression_cos_phi": 0.0002695527800824493, "Regression_energy": 0.04986296966671944, "Total": 2.1317317485809326, "Total_unweighted": 0.6228340863017365}, "train_group": {}, "valid": {"Classification_binary": 1.0890545845031738, "Classification": 0.009604080580174923, "Regression_pt": 0.0951722264289856, "Regression_eta": 0.00102297926787287, "Regression_sin_phi": 0.00803343951702118, "Regression_cos_phi": 0.003535780357196927, "Regression_energy": 0.09458310902118683, "Total": 12.92221450805664, "Total_unweighted": 1.3010061996756122, "metrics/particle/class_1/efficiency": 0.9748283624649048, "metrics/particle/class_1/pid_accuracy": 0.98591548204422, "metrics/particle/class_2/efficiency": 0.5185185074806213, "metrics/particle/class_2/pid_accuracy": 0.9285714030265808, "metrics/particle/class_3/efficiency": 0.8767676949501038, "metrics/particle/class_3/pid_accuracy": 0.9953917264938354, "metrics/particle/class_4/efficiency": 0.9491525292396545, "metrics/particle/class_4/pid_accuracy": 0.9553571343421936, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.00800000037997961, "metrics/particle/count/mae": 0.20800000429153442, "metrics/particle/count/prediction_mean": 4.639999866485596, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.040389202535152435, "metrics/particle/event/energy_response_mean": 0.9953290224075317, "metrics/particle/event/met_abs_error": 1.0743472576141357, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04067700356245041, "metrics/particle/event/scalar_pt_response_mean": 0.9940032362937927, "metrics/particle/event/vector_pt_closure": 0.041716210544109344, "metrics/particle/matched/delta_eta_abs_mean": 0.007157403044402599, "metrics/particle/matched/delta_phi_abs_mean": 0.007070775609463453, "metrics/particle/matched/delta_r_mean": 0.011228237301111221, "metrics/particle/matched/energy_relative_abs_mean": 0.061008960008621216, "metrics/particle/matched/pid_accuracy": 0.9869158864021301, "metrics/particle/matched/pt_relative_abs_mean": 0.061214011162519455, "metrics/particle/matching/duplicate_fraction": 0.01293103490024805, "metrics/particle/matching/efficiency": 0.9208261370658875, "metrics/particle/matching/f1": 0.9216192960739136, "metrics/particle/matching/purity": 0.9224137663841248, "metrics/particle/matching/target_coverage_dr0p05": 0.912220299243927, "metrics/particle/matching/target_coverage_dr0p10": 0.9466437101364136, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8511187434196472, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9406196475028992}, "step_train_time": 0.4672679901123047, "step_valid_time": 2.817164936000154, "step_total_time": 936.7747254371643, "elapsed_training_loop_seconds": 936.7747254371643, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9981479051679161, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.027666047622682988, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9883449883449883, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.986046511627907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.011655011655011656, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.986013986013986, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9837209302325581, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9837587006960556, "step_test_time": 4.141743051000049, "step_plot_time": 3.5347666410002603, "step_validation_cycle_time": 10.80462107800031} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_9000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_9000.json new file mode 100644 index 0000000000000000000000000000000000000000..0d58ffac59866809a86e00f103d7158edbded964 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/history/step_9000.json @@ -0,0 +1 @@ +{"epoch": 16.01423487544484, "train": {"Classification_binary": 0.4545533061027527, "Classification": 0.0006627130205743015, "Regression_pt": 0.011339973658323288, "Regression_eta": 0.00018030221690423787, "Regression_sin_phi": 0.00010036184539785609, "Regression_cos_phi": 9.39035089686513e-05, "Regression_energy": 0.01089700497686863, "Total": 0.9153029918670654, "Total_unweighted": 0.47782756532978965}, "train_group": {}, "valid": {"Classification_binary": 1.0821985006332397, "Classification": 0.009743898175656796, "Regression_pt": 0.09784392267465591, "Regression_eta": 0.0009940495947375894, "Regression_sin_phi": 0.0035603621508926153, "Regression_cos_phi": 0.0017728311941027641, "Regression_energy": 0.09780888259410858, "Total": 8.1332368850708, "Total_unweighted": 1.293922447017394, "metrics/particle/class_1/efficiency": 0.9748283624649048, "metrics/particle/class_1/pid_accuracy": 0.9812206625938416, "metrics/particle/class_2/efficiency": 0.5555555820465088, "metrics/particle/class_2/pid_accuracy": 0.800000011920929, "metrics/particle/class_3/efficiency": 0.8808080554008484, "metrics/particle/class_3/pid_accuracy": 0.9931192398071289, "metrics/particle/class_4/efficiency": 0.9491525292396545, "metrics/particle/class_4/pid_accuracy": 0.9553571343421936, "metrics/particle/class_5/efficiency": 0.9882352948188782, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.019999999552965164, "metrics/particle/count/mae": 0.19599999487400055, "metrics/particle/count/prediction_mean": 4.627999782562256, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.04406723752617836, "metrics/particle/event/energy_response_mean": 0.9962858557701111, "metrics/particle/event/met_abs_error": 1.0439183712005615, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04273897781968117, "metrics/particle/event/scalar_pt_response_mean": 0.9968113899230957, "metrics/particle/event/vector_pt_closure": 0.04204279184341431, "metrics/particle/matched/delta_eta_abs_mean": 0.007365177385509014, "metrics/particle/matched/delta_phi_abs_mean": 0.00665127020329237, "metrics/particle/matched/delta_r_mean": 0.011120522394776344, "metrics/particle/matched/energy_relative_abs_mean": 0.06390435993671417, "metrics/particle/matched/pid_accuracy": 0.982292652130127, "metrics/particle/matched/pt_relative_abs_mean": 0.06451434642076492, "metrics/particle/matching/duplicate_fraction": 0.012964563444256783, "metrics/particle/matching/efficiency": 0.9234079122543335, "metrics/particle/matching/f1": 0.9253988862037659, "metrics/particle/matching/purity": 0.9273984432220459, "metrics/particle/matching/target_coverage_dr0p05": 0.9087779521942139, "metrics/particle/matching/target_coverage_dr0p10": 0.9492254853248596, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.848537027835846, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9440619349479675}, "step_train_time": 0.45519161224365234, "step_valid_time": 2.819572780999806, "step_total_time": 1054.9394006729126, "elapsed_training_loop_seconds": 1054.9394006729126, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9993961225964405, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.030706157178846283, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 432, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9884259259259259, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9895712630359212, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.011574074074074073, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9861111111111112, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9872537659327926, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.182035251999878, "step_plot_time": 3.544405437000023, "step_validation_cycle_time": 10.859459395999693} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..c1ee9eb1c6c0b324053b4d19533f5ab8ff7d8f85 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2p5mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 30, "checkpoint_freq": 2000, "val_freq": 1000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", "relaxed_load": false, "load_weights_only": true, "reset_task_loss_weights": true, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_z_tautau_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..5a3046c378fab66e195ec747b2caf2510cd75dd4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24acfd4f92c1a405a9f7ab557c09c2a2f8005cac0d7383c56dc13b7f83e607f4 +size 4508 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..edd413c0c917f73d05383601a00b3077589a2821 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..bf60ce01cc4ad24e7b0cef1b3bf263e33e73cb1f Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..4bad05e966cb794cd3cfb689df59ace79f222ec8 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..ef274cf820a05b78776cecba313201a9645bd35a Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a1bdefb8956545f4c8abbc99c7e9ff5e1cdd5804 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..fad2e7b0dd2d98b1c3330b36279888b02b5cc929 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..01a1333a941fd63c79e403d1e84ed0debbe8181d Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..13e435f8b9b2b31c05cf8a59f7083ad1c769a45b Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..cb05b157574f6d301e4c428119af12f926d0caa3 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..58d2c2e4d22e02934cc42f54f3f40a71d4a9d8c7 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..802c638f7d07eba3b44222d34dc6e0174a4e7802 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..184bcc9b7e7c1bec9753dbca7613065b81d525d5 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/plots_step_35000/cld_edm_z_tautau_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/runs/train/events.out.tfevents.1794727308.workergpu169.143413.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/runs/train/events.out.tfevents.1794727308.workergpu169.143413.0 new file mode 100644 index 0000000000000000000000000000000000000000..d6c50a010f12f4eb8a0b091bda586145f27ceef4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/runs/train/events.out.tfevents.1794727308.workergpu169.143413.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d77d7d52525e23a152cb9626e7fd1a2df050bb89a5ea3c79142c67b6307f002f +size 1288219 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/runs/valid/events.out.tfevents.1794727308.workergpu169.143413.1 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/runs/valid/events.out.tfevents.1794727308.workergpu169.143413.1 new file mode 100644 index 0000000000000000000000000000000000000000..353f06018eac9776644a529c4e8826e46f46f2fc --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/runs/valid/events.out.tfevents.1794727308.workergpu169.143413.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eec54681d67227f67500a4e50d691c772273fbfcefd674c67906bbf8aef1f9aa +size 16140222 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..a3e3f1e91a5e11bb72d77005bac5e8b64c48eeba --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/scenario-manifest.json @@ -0,0 +1,1280 @@ +{ + "scenario": { + "name": "pf_fm_finetuning", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all" + }, + "allowed_variant_differences": [ + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets", + "data_config" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_finetuning", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_ecal_2p5mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2p5mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all", + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2p5mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 30, + "checkpoint_freq": 2000, + "val_freq": 1000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "relaxed_load": false, + "load_weights_only": true, + "reset_task_loss_weights": true, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_z_tautau_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2p5mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--load", + "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "1000", + "--checkpoint_freq", + "2000", + "--patience", + "30", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--load_weights_only", + "true", + "--reset_task_loss_weights", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.feature_bag", + "true", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.trainable", + "all", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2acbd5ca7a642ffffdc63cec3e3b8957dabf5421 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2p5mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_z_tautau_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +load_weights_only: true +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 30 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: true +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_z_tautau_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 1000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/train.log b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/train.log new file mode 100644 index 0000000000000000000000000000000000000000..2fcb3bc670aa73ad719b63a6aea112723f618610 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/train.log @@ -0,0 +1,2765 @@ +rank0 [2026-11-15 02:21:44,843] INFO : Will use single-gpu: NVIDIA H100 PCIe +rank0 [2026-11-15 02:21:44,844] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-15 02:21:45,906] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 02:21:45,906] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 02:21:45,907] INFO : Instantiating model +rank0 [2026-11-15 02:21:45,934] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 02:21:45,934] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 02:21:45,936] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 02:21:45,936] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 02:21:45,936] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 02:21:45,936] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 02:21:45,936] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,939] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 02:21:45,939] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,941] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 02:21:45,941] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,944] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 02:21:45,944] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,947] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 02:21:45,947] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,949] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 02:21:45,949] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,952] INFO : Backbone initialization took 0.02s +rank0 [2026-11-15 02:21:45,952] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 02:21:45,952] INFO : Initializing output DNNs +rank0 [2026-11-15 02:21:45,965] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,968] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,975] INFO : Output DNNs initialization took 0.02s +rank0 [2026-11-15 02:21:45,975] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 02:21:45,975] INFO : jagged_attention enabled=True +rank0 [2026-11-15 02:21:45,975] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 02:21:45,975] INFO : task_queries enabled=True +rank0 [2026-11-15 02:21:45,975] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 02:21:45,975] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 02:21:45,975] INFO : classification_query parameters: 256 +rank0 [2026-11-15 02:21:45,975] INFO : regression_query parameters: 256 +rank0 [2026-11-15 02:21:45,975] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 02:21:45,976] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 02:21:45,976] INFO : output_mode=elementwise +rank0 [2026-11-15 02:21:45,976] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 02:21:45,976] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 02:21:45,976] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 02:21:45,976] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 02:21:45,976] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 02:21:45,976] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 02:21:45,976] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 02:21:45,976] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 02:21:45,977] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 02:21:45,977] INFO : MLPF __init__ done +rank0 [2026-11-15 02:21:45,977] INFO : Instantiated model +rank0 [2026-11-15 02:21:45,978] INFO : Moving model to device rank=0 +rank0 [2026-11-15 02:21:45,989] INFO : Moved model to device rank=0 +rank0 [2026-11-15 02:21:45,990] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 02:21:46,117] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 02:21:46,117] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 02:21:46,117] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 02:21:46,215] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 02:21:46,215] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 02:21:46,216] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 02:21:46,216] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 02:21:46,216] INFO : Total parameters: 3910930 +rank0 [2026-11-15 02:21:46,218] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 02:21:46,224] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707 +rank0 [2026-11-15 02:21:46,224] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707 +rank0 [2026-11-15 02:21:46,224] INFO : Getting dataloaders +rank0 [2026-11-15 02:21:46,224] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:21:48,105] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,153] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 02:21:48,328] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 02:21:48,384] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,424] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,483] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,517] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,561] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,609] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 02:21:48,649] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,649] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 02:21:48,649] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 02:21:48,649] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-15 02:21:48,649] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:21:48,657] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,664] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,671] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,677] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,685] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,693] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,699] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,706] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,713] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,720] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,720] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 02:21:48,720] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 02:21:48,772] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:21:48,877] INFO : Got dataloaders +rank0 [2026-11-15 02:21:48,878] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-15 02:21:48,878] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-15 02:21:48,878] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 02:21:48,902] INFO : Creating train_iterator +rank0 [2026-11-15 02:21:48,902] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:21:48,924] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:21:49,037] INFO : Created train_iterator +rank0 [2026-11-15 02:22:01,450] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.643568235039711, 'Classification': 0.0195966562256217, 'Regression_pt': 0.17017275454476477, 'Regression_eta': 0.006485479580587708, 'Regression_sin_phi': 0.002063132655021036, 'Regression_cos_phi': 0.002265365930797998, 'Regression_energy': 0.17581136947497725}, weights={'Classification_binary': 1.0, 'Classification': 83.86982727050781, 'Regression_pt': 9.658233642578125, 'Regression_eta': 253.4227752685547, 'Regression_sin_phi': 796.63720703125, 'Regression_cos_phi': 725.5199584960938, 'Regression_energy': 9.348475456237793} +rank0 [2026-11-15 02:22:50,725] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 02:22:50,725] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:23:37,392] INFO : Step 1000/500000 rank0 | Train Loss: 5.3462 | Train Loss Unweighted: 1.1067 | Train Loss Components: Classification: 0.0076 | Classification_binary: 1.0055 | Regression_cos_phi: 0.0018 | Regression_energy: 0.0377 | Regression_eta: 0.0033 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.6337 | Classification_binary: 1.0055 | Regression_cos_phi: 1.3387 | Regression_energy: 0.3525 | Regression_eta: 0.8286 | Regression_pt: 0.4822 | Regression_sin_phi: 0.7050 | LR: 2.01e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4506s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0545 | max_tensorboard_logging: 0.1462 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2582.1 | uss_MB: 1347.2 | child_rss_MB: 22995.0 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:23:37,394] INFO : Running validation on rank0 +rank0 [2026-11-15 02:23:37,394] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:23:37,394] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:23:37,592] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 02:23:37,793] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:23:37,862] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:23:37,916] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:23:37,970] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:23:38,016] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:23:38,068] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:23:38,121] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:23:38,170] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:23:38,248] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:23:38,321] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:23:38,372] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:23:38,425] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:23:38,469] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:23:38,522] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:23:38,575] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:23:38,621] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:23:38,625] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:23:38,627] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:23:38,629] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:23:38,631] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:23:38,633] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:23:38,633] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:23:38,635] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:23:38,637] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:23:38,638] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:23:38,639] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:23:38,641] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:23:38,643] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:23:38,645] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:23:38,647] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:23:38,649] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:23:40,086] INFO : VALIDATION | Step=1000/500000 | Train Loss=5.3462 | Valid Loss=8.8955 | Train Loss Unweighted=1.1067 | Valid Loss Unweighted=1.3492 | Stale=0 | ETA=924.7m +rank0 [2026-11-15 02:23:40,087] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:23:40,162] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:23:40,177] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:23:40,227] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:23:40,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:23:41,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:23:41,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:23:41,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:23:41,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:23:42,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:23:42,383] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:23:42,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:23:42,850] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:23:43,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:23:43,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:23:43,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:23:43,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:23:44,054] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:23:44,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:23:44,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:23:44,556] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:23:44,562] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:23:45,087] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:23:46,006] INFO : Plotted number of elements +rank0 [2026-11-15 02:23:47,276] INFO : Plotted jets +rank0 [2026-11-15 02:23:48,615] INFO : Plotted jet ratio +rank0 [2026-11-15 02:24:02,169] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 02:24:02,169] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:25:01,908] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 02:25:01,908] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:25:35,650] INFO : Step 2000/500000 rank0 | Train Loss: 2.7241 | Train Loss Unweighted: 0.9196 | Train Loss Components: Classification: 0.0104 | Classification_binary: 0.8333 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0375 | Regression_eta: 0.0002 | Regression_pt: 0.0379 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.8682 | Classification_binary: 0.8333 | Regression_cos_phi: 0.0698 | Regression_energy: 0.3502 | Regression_eta: 0.0582 | Regression_pt: 0.3659 | Regression_sin_phi: 0.1786 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4456s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0033 | max_backward: 0.0393 | max_optimizer_step: 0.0589 | max_tensorboard_logging: 0.1277 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2714.9 | uss_MB: 1486.6 | child_rss_MB: 22765.0 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:25:35,652] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 02:25:35,716] INFO : Running validation on rank0 +rank0 [2026-11-15 02:25:35,716] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:25:35,716] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:25:35,836] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 02:25:35,940] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:25:36,041] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:25:36,111] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:25:36,177] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:25:36,240] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:25:36,305] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:25:36,370] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:25:36,479] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:25:36,589] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:25:36,698] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:25:36,765] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:25:36,830] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:25:36,894] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:25:36,959] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:25:37,023] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:25:37,135] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:25:37,142] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:25:37,144] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:25:37,146] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:25:37,148] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:25:37,150] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:25:37,151] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:25:37,153] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:25:37,154] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:25:37,155] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:25:37,157] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:25:37,159] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:25:37,160] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:25:37,161] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:25:37,162] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:25:37,165] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:25:38,544] INFO : VALIDATION | Step=2000/500000 | Train Loss=2.7241 | Valid Loss=10.2432 | Train Loss Unweighted=0.9196 | Valid Loss Unweighted=1.2907 | Stale=1 | ETA=953.0m +rank0 [2026-11-15 02:25:38,545] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:25:38,617] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:25:38,633] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:25:38,668] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:25:39,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:25:39,329] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:25:39,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:25:39,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:25:40,038] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:25:40,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:25:40,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:25:40,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:25:40,963] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:25:41,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:25:41,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:25:41,684] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:25:41,919] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:25:42,153] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:25:42,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:25:42,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:25:42,632] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:25:42,637] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:25:43,167] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:25:43,554] INFO : Plotted number of elements +rank0 [2026-11-15 02:25:44,786] INFO : Plotted jets +rank0 [2026-11-15 02:25:46,096] INFO : Plotted jet ratio +rank0 [2026-11-15 02:26:12,842] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 02:26:12,842] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:27:12,639] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 02:27:12,639] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:27:33,292] INFO : Step 3000/500000 rank0 | Train Loss: 3.3016 | Train Loss Unweighted: 0.8372 | Train Loss Components: Classification: 0.0021 | Classification_binary: 0.7160 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0623 | Regression_eta: 0.0026 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.1773 | Classification_binary: 0.7160 | Regression_cos_phi: 0.2194 | Regression_energy: 0.5820 | Regression_eta: 0.6629 | Regression_pt: 0.5155 | Regression_sin_phi: 0.4284 | LR: 2.05e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4424s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0027 | max_backward: 0.0385 | max_optimizer_step: 0.0569 | max_tensorboard_logging: 0.1270 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2773.8 | uss_MB: 1545.6 | child_rss_MB: 22765.1 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:27:33,293] INFO : Running validation on rank0 +rank0 [2026-11-15 02:27:33,294] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:27:33,294] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:27:33,414] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 02:27:33,520] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:27:33,618] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:27:33,686] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:27:33,750] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:27:33,815] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:27:33,879] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:27:33,945] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:27:34,053] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:27:34,163] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:27:34,274] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:27:34,342] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:27:34,408] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:27:34,473] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:27:34,542] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:27:34,606] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:27:34,714] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:27:34,720] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:27:34,722] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:27:34,724] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:27:34,726] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:27:34,727] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:27:34,729] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:27:34,731] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:27:34,731] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:27:34,733] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:27:34,735] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:27:34,737] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:27:34,738] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:27:34,740] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:27:34,742] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:27:34,744] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:27:36,134] INFO : VALIDATION | Step=3000/500000 | Train Loss=3.3016 | Valid Loss=14.6085 | Train Loss Unweighted=0.8372 | Valid Loss Unweighted=1.2769 | Stale=2 | ETA=958.7m +rank0 [2026-11-15 02:27:36,134] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:27:36,206] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:27:36,209] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:27:36,249] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:27:36,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:27:36,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:27:37,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:27:37,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:27:37,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:27:37,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:27:38,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:27:38,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:27:38,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:27:38,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:27:39,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:27:39,301] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:27:39,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:27:39,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:27:40,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:27:40,226] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:27:40,277] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:27:40,282] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:27:40,814] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:27:41,214] INFO : Plotted number of elements +rank0 [2026-11-15 02:27:42,434] INFO : Plotted jets +rank0 [2026-11-15 02:27:43,762] INFO : Plotted jet ratio +rank0 [2026-11-15 02:28:23,908] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 02:28:23,908] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:29:24,040] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 02:29:24,040] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:29:31,467] INFO : Step 4000/500000 rank0 | Train Loss: 4.8670 | Train Loss Unweighted: 1.8491 | Train Loss Components: Classification: 0.0035 | Classification_binary: 1.6030 | Regression_cos_phi: 0.0003 | Regression_energy: 0.1218 | Regression_eta: 0.0004 | Regression_pt: 0.1197 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.2936 | Classification_binary: 1.6030 | Regression_cos_phi: 0.2408 | Regression_energy: 1.1384 | Regression_eta: 0.1056 | Regression_pt: 1.1559 | Regression_sin_phi: 0.3297 | LR: 2.08e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4465s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0571 | max_tensorboard_logging: 0.1308 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2803.7 | uss_MB: 1575.5 | child_rss_MB: 22765.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:29:31,469] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 02:29:31,538] INFO : Running validation on rank0 +rank0 [2026-11-15 02:29:31,538] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:29:31,538] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:29:31,657] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 02:29:31,761] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:29:31,859] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:29:31,925] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:29:31,989] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:29:32,054] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:29:32,119] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:29:32,185] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:29:32,292] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:29:32,395] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:29:32,496] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:29:32,562] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:29:32,627] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:29:32,690] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:29:32,755] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:29:32,820] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:29:32,931] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:29:32,938] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:29:32,940] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:29:32,942] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:29:32,944] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:29:32,946] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:29:32,947] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:29:32,948] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:29:32,950] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:29:32,951] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:29:32,953] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:29:32,955] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:29:32,957] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:29:32,959] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:29:32,960] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:29:32,963] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:29:34,357] INFO : VALIDATION | Step=4000/500000 | Train Loss=4.8670 | Valid Loss=10.7381 | Train Loss Unweighted=1.8491 | Valid Loss Unweighted=1.2477 | Stale=3 | ETA=961.9m +rank0 [2026-11-15 02:29:34,357] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:29:34,432] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:29:34,436] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:29:34,477] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:29:34,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:29:35,129] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:29:35,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:29:35,607] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:29:35,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:29:36,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:29:36,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:29:36,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:29:36,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:29:37,028] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:29:37,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:29:37,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:29:37,739] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:29:37,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:29:38,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:29:38,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:29:38,480] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:29:38,485] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:29:39,011] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:29:39,410] INFO : Plotted number of elements +rank0 [2026-11-15 02:29:40,635] INFO : Plotted jets +rank0 [2026-11-15 02:29:41,988] INFO : Plotted jet ratio +rank0 [2026-11-15 02:30:34,990] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 02:30:34,990] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:31:29,085] INFO : Step 5000/500000 rank0 | Train Loss: 2.2699 | Train Loss Unweighted: 0.8798 | Train Loss Components: Classification: 0.0030 | Classification_binary: 0.8234 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0276 | Regression_eta: 0.0006 | Regression_pt: 0.0245 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.2484 | Classification_binary: 0.8234 | Regression_cos_phi: 0.4323 | Regression_energy: 0.2583 | Regression_eta: 0.1421 | Regression_pt: 0.2365 | Regression_sin_phi: 0.1288 | LR: 2.13e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4464s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0345 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0569 | max_tensorboard_logging: 0.1286 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2826.4 | uss_MB: 1598.3 | child_rss_MB: 22765.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:31:29,087] INFO : Running validation on rank0 +rank0 [2026-11-15 02:31:29,087] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:31:29,087] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:31:29,214] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 02:31:29,320] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:31:29,423] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:31:29,491] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:31:29,558] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:31:29,621] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:31:29,685] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:31:29,750] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:31:29,859] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:31:29,966] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:31:30,076] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:31:30,144] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:31:30,209] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:31:30,272] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:31:30,337] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:31:30,401] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:31:30,511] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:31:30,517] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:31:30,519] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:31:30,521] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:31:30,523] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:31:30,525] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:31:30,526] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:31:30,526] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:31:30,529] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:31:30,530] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:31:30,532] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:31:30,534] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:31:30,534] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:31:30,536] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:31:30,538] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:31:30,540] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:31:32,026] INFO : VALIDATION | Step=5000/500000 | Train Loss=2.2699 | Valid Loss=8.0372 | Train Loss Unweighted=0.8798 | Valid Loss Unweighted=1.2434 | Stale=0 | ETA=962.2m +rank0 [2026-11-15 02:31:32,026] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:31:32,100] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:31:32,103] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:31:32,149] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:31:32,607] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:31:32,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:31:33,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:31:33,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:31:33,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:31:33,794] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:31:34,028] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:31:34,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:31:34,492] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:31:34,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:31:34,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:31:35,198] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:31:35,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:31:35,673] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:31:35,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:31:36,133] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:31:36,188] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:31:36,193] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:31:36,716] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:31:37,115] INFO : Plotted number of elements +rank0 [2026-11-15 02:31:38,351] INFO : Plotted jets +rank0 [2026-11-15 02:31:39,690] INFO : Plotted jet ratio +rank0 [2026-11-15 02:31:46,241] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 02:31:46,241] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:32:46,501] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 02:32:46,502] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:33:27,382] INFO : Step 6000/500000 rank0 | Train Loss: 2.4642 | Train Loss Unweighted: 0.9994 | Train Loss Components: Classification: 0.0008 | Classification_binary: 0.8981 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0387 | Regression_eta: 0.0010 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.8981 | Regression_cos_phi: 0.1441 | Regression_energy: 0.3621 | Regression_eta: 0.2505 | Regression_pt: 0.5822 | Regression_sin_phi: 0.1565 | LR: 2.19e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4441s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0026 | max_backward: 0.0379 | max_optimizer_step: 0.0564 | max_tensorboard_logging: 0.1262 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2852.8 | uss_MB: 1624.6 | child_rss_MB: 22765.2 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:33:27,383] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 02:33:27,452] INFO : Running validation on rank0 +rank0 [2026-11-15 02:33:27,452] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:33:27,452] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:33:27,571] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 02:33:27,678] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:33:27,780] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:33:27,848] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:33:27,915] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:33:27,981] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:33:28,047] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:33:28,116] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:33:28,226] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:33:28,329] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:33:28,432] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:33:28,498] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:33:28,563] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:33:28,626] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:33:28,690] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:33:28,754] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:33:28,848] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:33:28,854] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:33:28,856] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:33:28,858] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:33:28,860] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:33:28,862] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:33:28,863] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:33:28,865] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:33:28,866] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:33:28,867] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:33:28,869] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:33:28,871] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:33:28,873] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:33:28,875] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:33:28,876] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:33:28,879] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:33:30,273] INFO : VALIDATION | Step=6000/500000 | Train Loss=2.4642 | Valid Loss=10.8536 | Train Loss Unweighted=0.9994 | Valid Loss Unweighted=1.2972 | Stale=1 | ETA=962.4m +rank0 [2026-11-15 02:33:30,274] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:33:30,346] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:33:30,349] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:33:30,391] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:33:30,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:33:31,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:33:31,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:33:31,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:33:31,771] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:33:32,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:33:32,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:33:32,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:33:32,721] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:33:32,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:33:33,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:33:33,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:33:33,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:33:33,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:33:34,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:33:34,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:33:34,414] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:33:34,419] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:33:34,945] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:33:35,332] INFO : Plotted number of elements +rank0 [2026-11-15 02:33:36,572] INFO : Plotted jets +rank0 [2026-11-15 02:33:37,931] INFO : Plotted jet ratio +rank0 [2026-11-15 02:33:57,588] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 02:33:57,588] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:34:57,436] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 02:34:57,436] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:35:25,025] INFO : Step 7000/500000 rank0 | Train Loss: 2.1415 | Train Loss Unweighted: 0.9811 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.8995 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0393 | Regression_eta: 0.0004 | Regression_pt: 0.0413 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0178 | Classification_binary: 0.8995 | Regression_cos_phi: 0.1180 | Regression_energy: 0.3674 | Regression_eta: 0.0957 | Regression_pt: 0.3986 | Regression_sin_phi: 0.2446 | LR: 2.26e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4570s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0354 | max_loss: 0.0027 | max_backward: 0.0398 | max_optimizer_step: 0.0602 | max_tensorboard_logging: 0.1289 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2874.5 | uss_MB: 1646.4 | child_rss_MB: 22764.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 02:35:25,027] INFO : Running validation on rank0 +rank0 [2026-11-15 02:35:25,027] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:35:25,027] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:35:25,143] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 02:35:25,246] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:35:25,346] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:35:25,414] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:35:25,478] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:35:25,542] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:35:25,607] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:35:25,672] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:35:25,779] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:35:25,883] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:35:25,987] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:35:26,054] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:35:26,119] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:35:26,184] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:35:26,250] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:35:26,316] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:35:26,412] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:35:26,418] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:35:26,420] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:35:26,422] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:35:26,424] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:35:26,426] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:35:26,427] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:35:26,427] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:35:26,430] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:35:26,431] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:35:26,433] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:35:26,435] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:35:26,436] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:35:26,437] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:35:26,439] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:35:26,441] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:35:27,825] INFO : VALIDATION | Step=7000/500000 | Train Loss=2.1415 | Valid Loss=11.9212 | Train Loss Unweighted=0.9811 | Valid Loss Unweighted=1.2719 | Stale=2 | ETA=961.3m +rank0 [2026-11-15 02:35:27,825] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:35:27,900] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:35:27,904] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:35:27,947] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:35:28,404] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:35:28,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:35:28,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:35:29,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:35:29,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:35:29,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:35:29,845] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:35:30,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:35:30,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:35:30,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:35:30,794] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:35:31,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:35:31,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:35:31,510] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:35:31,741] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:35:31,955] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:35:32,014] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:35:32,019] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:35:32,549] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:35:32,947] INFO : Plotted number of elements +rank0 [2026-11-15 02:35:34,575] INFO : Plotted jets +rank0 [2026-11-15 02:35:35,491] INFO : Plotted jet ratio +rank0 [2026-11-15 02:36:08,461] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 02:36:08,461] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:37:08,343] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 02:37:08,343] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:37:22,788] INFO : Step 8000/500000 rank0 | Train Loss: 2.1317 | Train Loss Unweighted: 0.6228 | Train Loss Components: Classification: 0.0021 | Classification_binary: 0.5271 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0499 | Regression_eta: 0.0009 | Regression_pt: 0.0425 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.1748 | Classification_binary: 0.5271 | Regression_cos_phi: 0.1956 | Regression_energy: 0.4661 | Regression_eta: 0.2387 | Regression_pt: 0.4102 | Regression_sin_phi: 0.1192 | LR: 2.34e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4667s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0373 | max_loss: 0.0030 | max_backward: 0.0428 | max_optimizer_step: 0.0615 | max_tensorboard_logging: 0.1272 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2899.4 | uss_MB: 1671.2 | child_rss_MB: 22763.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:37:22,790] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 02:37:22,858] INFO : Running validation on rank0 +rank0 [2026-11-15 02:37:22,858] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:37:22,858] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:37:22,974] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 02:37:23,079] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:37:23,177] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:37:23,245] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:37:23,311] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:37:23,375] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:37:23,441] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:37:23,507] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:37:23,618] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:37:23,729] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:37:23,837] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:37:23,905] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:37:23,970] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:37:24,036] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:37:24,103] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:37:24,169] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:37:24,263] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:37:24,269] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:37:24,271] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:37:24,273] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:37:24,275] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:37:24,277] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:37:24,277] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:37:24,279] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:37:24,281] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:37:24,282] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:37:24,284] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:37:24,285] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:37:24,288] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:37:24,289] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:37:24,291] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:37:24,293] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:37:25,679] INFO : VALIDATION | Step=8000/500000 | Train Loss=2.1317 | Valid Loss=12.9222 | Train Loss Unweighted=0.6228 | Valid Loss Unweighted=1.3010 | Stale=3 | ETA=960.2m +rank0 [2026-11-15 02:37:25,679] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:37:25,750] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:37:25,753] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:37:25,792] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:37:26,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:37:26,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:37:26,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:37:26,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:37:27,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:37:27,417] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:37:27,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:37:27,884] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:37:28,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:37:28,355] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:37:28,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:37:28,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:37:29,077] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:37:29,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:37:29,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:37:29,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:37:29,819] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:37:29,824] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:37:30,354] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:37:30,759] INFO : Plotted number of elements +rank0 [2026-11-15 02:37:31,998] INFO : Plotted jets +rank0 [2026-11-15 02:37:33,346] INFO : Plotted jet ratio +rank0 [2026-11-15 02:38:19,841] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 02:38:19,841] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:39:19,807] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 02:39:19,807] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:39:21,019] INFO : Step 9000/500000 rank0 | Train Loss: 0.9153 | Train Loss Unweighted: 0.4778 | Train Loss Components: Classification: 0.0007 | Classification_binary: 0.4546 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0109 | Regression_eta: 0.0002 | Regression_pt: 0.0113 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.4546 | Regression_cos_phi: 0.0681 | Regression_energy: 0.1019 | Regression_eta: 0.0457 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0800 | LR: 2.43e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4549s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0575 | max_tensorboard_logging: 0.1371 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2853.0 | uss_MB: 1624.9 | child_rss_MB: 22759.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:39:21,020] INFO : Running validation on rank0 +rank0 [2026-11-15 02:39:21,020] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:39:21,020] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:39:21,156] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 02:39:21,262] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:39:21,361] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:39:21,430] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:39:21,495] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:39:21,558] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:39:21,622] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:39:21,687] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:39:21,795] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:39:21,903] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:39:22,009] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:39:22,076] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:39:22,140] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:39:22,204] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:39:22,269] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:39:22,335] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:39:22,431] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:39:22,436] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:39:22,438] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:39:22,440] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:39:22,442] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:39:22,444] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:39:22,445] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:39:22,447] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:39:22,449] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:39:22,449] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:39:22,451] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:39:22,453] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:39:22,454] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:39:22,454] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:39:22,457] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:39:22,460] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:39:23,843] INFO : VALIDATION | Step=9000/500000 | Train Loss=0.9153 | Valid Loss=8.1332 | Train Loss Unweighted=0.4778 | Valid Loss Unweighted=1.2939 | Stale=4 | ETA=959.2m +rank0 [2026-11-15 02:39:23,844] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:39:23,915] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:39:23,929] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:39:23,972] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:39:24,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:39:24,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:39:24,906] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:39:25,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:39:25,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:39:25,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:39:25,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:39:26,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:39:26,329] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:39:26,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:39:26,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:39:27,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:39:27,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:39:27,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:39:27,759] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:39:27,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:39:28,024] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:39:28,041] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:39:28,566] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:39:28,982] INFO : Plotted number of elements +rank0 [2026-11-15 02:39:30,221] INFO : Plotted jets +rank0 [2026-11-15 02:39:31,562] INFO : Plotted jet ratio +rank0 [2026-11-15 02:40:30,948] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 02:40:30,948] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:41:18,920] INFO : Step 10000/500000 rank0 | Train Loss: 1.1075 | Train Loss Unweighted: 0.4842 | Train Loss Components: Classification: 0.0008 | Classification_binary: 0.4421 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0206 | Regression_eta: 0.0001 | Regression_pt: 0.0204 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.4421 | Regression_cos_phi: 0.1057 | Regression_energy: 0.1922 | Regression_eta: 0.0307 | Regression_pt: 0.1970 | Regression_sin_phi: 0.0768 | LR: 2.52e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4574s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0387 | max_optimizer_step: 0.0591 | max_tensorboard_logging: 0.1275 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2950.1 | uss_MB: 1722.0 | child_rss_MB: 22762.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:41:18,922] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 02:41:18,989] INFO : Running validation on rank0 +rank0 [2026-11-15 02:41:18,989] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:41:18,989] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:41:19,118] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 02:41:19,223] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:41:19,327] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:41:19,394] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:41:19,459] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:41:19,523] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:41:19,591] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:41:19,657] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:41:19,765] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:41:19,868] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:41:19,972] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:41:20,038] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:41:20,102] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:41:20,166] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:41:20,231] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:41:20,296] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:41:20,393] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:41:20,399] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:41:20,401] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:41:20,403] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:41:20,405] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:41:20,407] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:41:20,408] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:41:20,409] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:41:20,411] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:41:20,412] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:41:20,414] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:41:20,416] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:41:20,418] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:41:20,419] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:41:20,421] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:41:20,423] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:41:21,836] INFO : VALIDATION | Step=10000/500000 | Train Loss=1.1075 | Valid Loss=8.1656 | Train Loss Unweighted=0.4842 | Valid Loss Unweighted=1.3168 | Stale=5 | ETA=957.9m +rank0 [2026-11-15 02:41:21,836] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:41:21,910] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:41:21,914] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:41:21,955] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:41:22,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:41:22,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:41:22,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:41:23,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:41:23,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:41:23,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:41:23,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:41:24,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:41:24,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:41:24,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:41:24,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:41:25,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:41:25,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:41:25,476] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:41:25,712] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:41:25,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:41:25,988] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:41:25,994] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:41:26,517] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:41:26,914] INFO : Plotted number of elements +rank0 [2026-11-15 02:41:28,137] INFO : Plotted jets +rank0 [2026-11-15 02:41:29,466] INFO : Plotted jet ratio +rank0 [2026-11-15 02:41:42,151] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 02:41:42,151] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:42:42,070] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 02:42:42,070] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:43:16,809] INFO : Step 11000/500000 rank0 | Train Loss: 1.0296 | Train Loss Unweighted: 0.3118 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2535 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0288 | Regression_eta: 0.0001 | Regression_pt: 0.0286 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.2535 | Regression_cos_phi: 0.0514 | Regression_energy: 0.2697 | Regression_eta: 0.0301 | Regression_pt: 0.2766 | Regression_sin_phi: 0.1106 | LR: 2.63e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4653s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0026 | max_backward: 0.0385 | max_optimizer_step: 0.0591 | max_tensorboard_logging: 0.1347 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2977.2 | uss_MB: 1749.1 | child_rss_MB: 22767.2 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 12.0 +rank0 [2026-11-15 02:43:16,811] INFO : Running validation on rank0 +rank0 [2026-11-15 02:43:16,811] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:43:16,811] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:43:16,943] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 02:43:17,048] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:43:17,151] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:43:17,221] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:43:17,287] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:43:17,353] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:43:17,420] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:43:17,487] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:43:17,596] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:43:17,699] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:43:17,801] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:43:17,867] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:43:17,932] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:43:17,994] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:43:18,058] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:43:18,122] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:43:18,216] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:43:18,221] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:43:18,223] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:43:18,224] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:43:18,228] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:43:18,229] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:43:18,230] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:43:18,232] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:43:18,234] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:43:18,234] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:43:18,236] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:43:18,238] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:43:18,239] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:43:18,239] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:43:18,243] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:43:18,245] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:43:19,644] INFO : VALIDATION | Step=11000/500000 | Train Loss=1.0296 | Valid Loss=14.4730 | Train Loss Unweighted=0.3118 | Valid Loss Unweighted=1.3514 | Stale=6 | ETA=956.3m +rank0 [2026-11-15 02:43:19,645] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:43:19,721] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:43:19,724] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:43:19,763] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:43:20,189] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:43:20,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:43:20,670] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:43:20,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:43:21,142] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:43:21,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:43:21,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:43:21,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:43:22,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:43:22,337] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:43:22,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:43:22,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:43:23,043] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:43:23,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:43:23,526] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:43:23,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:43:23,802] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:43:23,808] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:43:24,335] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:43:24,723] INFO : Plotted number of elements +rank0 [2026-11-15 02:43:25,970] INFO : Plotted jets +rank0 [2026-11-15 02:43:27,305] INFO : Plotted jet ratio +rank0 [2026-11-15 02:43:53,316] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 02:43:53,316] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:44:53,514] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 02:44:53,514] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:45:15,085] INFO : Step 12000/500000 rank0 | Train Loss: 1.4650 | Train Loss Unweighted: 0.3714 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.3256 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0219 | Regression_eta: 0.0002 | Regression_pt: 0.0226 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0219 | Classification_binary: 0.3256 | Regression_cos_phi: 0.4388 | Regression_energy: 0.2044 | Regression_eta: 0.0486 | Regression_pt: 0.2181 | Regression_sin_phi: 0.2076 | LR: 2.75e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4528s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0579 | max_tensorboard_logging: 0.1269 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2984.0 | uss_MB: 1755.8 | child_rss_MB: 22766.1 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:45:15,086] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 02:45:15,152] INFO : Running validation on rank0 +rank0 [2026-11-15 02:45:15,152] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:45:15,152] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:45:15,273] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 02:45:15,379] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:45:15,480] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:45:15,549] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:45:15,614] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:45:15,679] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:45:15,747] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:45:15,814] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:45:15,923] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:45:16,026] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:45:16,128] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:45:16,195] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:45:16,258] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:45:16,322] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:45:16,387] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:45:16,452] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:45:16,546] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:45:16,552] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:45:16,554] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:45:16,556] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:45:16,558] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:45:16,560] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:45:16,561] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:45:16,561] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:45:16,565] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:45:16,565] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:45:16,567] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:45:16,569] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:45:16,570] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:45:16,570] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:45:16,574] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:45:16,576] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:45:17,970] INFO : VALIDATION | Step=12000/500000 | Train Loss=1.4650 | Valid Loss=11.5311 | Train Loss Unweighted=0.3714 | Valid Loss Unweighted=1.3716 | Stale=7 | ETA=955.0m +rank0 [2026-11-15 02:45:17,970] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:45:18,045] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:45:18,061] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:45:18,105] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:45:18,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:45:18,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:45:19,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:45:19,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:45:19,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:45:19,766] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:45:20,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:45:20,256] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:45:20,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:45:20,739] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:45:20,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:45:21,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:45:21,448] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:45:21,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:45:21,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:45:22,153] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:45:22,196] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:45:22,203] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:45:22,737] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:45:23,130] INFO : Plotted number of elements +rank0 [2026-11-15 02:45:24,389] INFO : Plotted jets +rank0 [2026-11-15 02:45:25,725] INFO : Plotted jet ratio +rank0 [2026-11-15 02:46:05,159] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 02:46:05,159] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:47:05,252] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 02:47:05,252] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:47:13,574] INFO : Step 13000/500000 rank0 | Train Loss: 1.7463 | Train Loss Unweighted: 0.8769 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.8029 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0367 | Regression_eta: 0.0002 | Regression_pt: 0.0366 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0264 | Classification_binary: 0.8029 | Regression_cos_phi: 0.0661 | Regression_energy: 0.3427 | Regression_eta: 0.0526 | Regression_pt: 0.3537 | Regression_sin_phi: 0.1019 | LR: 2.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4741s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0382 | max_optimizer_step: 0.0590 | max_tensorboard_logging: 0.1260 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3019.2 | uss_MB: 1791.1 | child_rss_MB: 22765.9 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:47:13,576] INFO : Running validation on rank0 +rank0 [2026-11-15 02:47:13,576] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:47:13,576] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:47:13,705] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 02:47:13,813] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:47:13,912] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:47:13,981] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:47:14,046] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:47:14,109] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:47:14,174] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:47:14,239] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:47:14,350] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:47:14,459] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:47:14,569] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:47:14,638] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:47:14,704] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:47:14,769] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:47:14,835] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:47:14,900] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:47:14,995] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:47:15,001] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:47:15,001] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:47:15,005] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:47:15,005] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:47:15,007] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:47:15,009] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:47:15,011] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:47:15,013] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:47:15,014] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:47:15,016] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:47:15,018] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:47:15,020] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:47:15,022] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:47:15,024] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:47:15,026] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:47:16,421] INFO : VALIDATION | Step=13000/500000 | Train Loss=1.7463 | Valid Loss=14.5748 | Train Loss Unweighted=0.8769 | Valid Loss Unweighted=1.3713 | Stale=8 | ETA=953.7m +rank0 [2026-11-15 02:47:16,422] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:47:16,492] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:47:16,497] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:47:16,540] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:47:16,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:47:17,212] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:47:17,447] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:47:17,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:47:17,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:47:18,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:47:18,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:47:18,635] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:47:18,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:47:19,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:47:19,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:47:19,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:47:19,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:47:20,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:47:20,290] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:47:20,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:47:20,562] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:47:20,568] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:47:21,090] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:47:21,486] INFO : Plotted number of elements +rank0 [2026-11-15 02:47:22,710] INFO : Plotted jets +rank0 [2026-11-15 02:47:24,049] INFO : Plotted jet ratio +rank0 [2026-11-15 02:48:16,272] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 02:48:16,272] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:49:11,172] INFO : Step 14000/500000 rank0 | Train Loss: 3.6761 | Train Loss Unweighted: 0.6192 | Train Loss Components: Classification: 0.0010 | Classification_binary: 0.3692 | Regression_cos_phi: 0.0001 | Regression_energy: 0.1224 | Regression_eta: 0.0028 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0801 | Classification_binary: 0.3692 | Regression_cos_phi: 0.1063 | Regression_energy: 1.1441 | Regression_eta: 0.7178 | Regression_pt: 1.1942 | Regression_sin_phi: 0.0644 | LR: 3.02e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4606s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0343 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0590 | max_tensorboard_logging: 0.1320 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3051.1 | uss_MB: 1823.0 | child_rss_MB: 22766.5 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 02:49:11,178] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 02:49:11,244] INFO : Running validation on rank0 +rank0 [2026-11-15 02:49:11,244] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:49:11,244] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:49:11,365] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 02:49:11,474] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:49:11,576] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:49:11,646] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:49:11,713] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:49:11,778] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:49:11,845] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:49:11,912] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:49:12,023] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:49:12,128] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:49:12,230] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:49:12,297] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:49:12,362] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:49:12,426] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:49:12,490] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:49:12,555] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:49:12,649] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:49:12,655] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:49:12,657] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:49:12,660] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:49:12,661] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:49:12,663] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:49:12,664] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:49:12,666] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:49:12,668] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:49:12,668] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:49:12,670] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:49:12,672] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:49:12,673] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:49:12,673] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:49:12,675] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:49:12,679] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:49:14,069] INFO : VALIDATION | Step=14000/500000 | Train Loss=3.6761 | Valid Loss=14.8881 | Train Loss Unweighted=0.6192 | Valid Loss Unweighted=1.4286 | Stale=9 | ETA=951.8m +rank0 [2026-11-15 02:49:14,069] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:49:14,145] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:49:14,148] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:49:14,190] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:49:14,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:49:14,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:49:15,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:49:15,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:49:15,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:49:15,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:49:16,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:49:16,307] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:49:16,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:49:16,789] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:49:17,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:49:17,260] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:49:17,496] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:49:17,733] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:49:17,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:49:18,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:49:18,247] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:49:18,254] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:49:18,778] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:49:19,165] INFO : Plotted number of elements +rank0 [2026-11-15 02:49:20,425] INFO : Plotted jets +rank0 [2026-11-15 02:49:21,836] INFO : Plotted jet ratio +rank0 [2026-11-15 02:49:27,466] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 02:49:27,466] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:50:27,331] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 02:50:27,332] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:51:08,968] INFO : Step 15000/500000 rank0 | Train Loss: 0.6973 | Train Loss Unweighted: 0.1694 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1410 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0137 | Regression_eta: 0.0003 | Regression_pt: 0.0141 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0015 | Classification_binary: 0.1410 | Regression_cos_phi: 0.0974 | Regression_energy: 0.1279 | Regression_eta: 0.0735 | Regression_pt: 0.1364 | Regression_sin_phi: 0.1196 | LR: 3.17e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4613s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0369 | max_loss: 0.0031 | max_backward: 0.0390 | max_optimizer_step: 0.0604 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3072.5 | uss_MB: 1844.4 | child_rss_MB: 22765.2 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:51:08,970] INFO : Running validation on rank0 +rank0 [2026-11-15 02:51:08,970] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:51:08,970] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:51:09,087] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 02:51:09,191] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:51:09,290] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:51:09,359] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:51:09,424] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:51:09,488] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:51:09,554] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:51:09,619] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:51:09,731] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:51:09,842] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:51:09,952] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:51:10,019] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:51:10,085] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:51:10,151] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:51:10,217] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:51:10,281] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:51:10,376] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:51:10,382] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:51:10,384] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:51:10,386] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:51:10,388] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:51:10,390] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:51:10,391] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:51:10,393] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:51:10,395] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:51:10,395] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:51:10,397] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:51:10,399] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:51:10,401] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:51:10,403] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:51:10,405] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:51:10,407] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:51:11,788] INFO : VALIDATION | Step=15000/500000 | Train Loss=0.6973 | Valid Loss=10.6270 | Train Loss Unweighted=0.1694 | Valid Loss Unweighted=1.5030 | Stale=10 | ETA=950.0m +rank0 [2026-11-15 02:51:11,788] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:51:11,863] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:51:11,866] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:51:11,911] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:51:12,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:51:12,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:51:12,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:51:13,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:51:13,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:51:13,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:51:13,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:51:14,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:51:14,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:51:14,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:51:14,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:51:15,038] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:51:15,275] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:51:15,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:51:15,741] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:51:15,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:51:16,008] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:51:16,013] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:51:16,550] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:51:16,939] INFO : Plotted number of elements +rank0 [2026-11-15 02:51:18,174] INFO : Plotted jets +rank0 [2026-11-15 02:51:19,515] INFO : Plotted jet ratio +rank0 [2026-11-15 02:51:38,472] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 02:51:38,473] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:52:38,661] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 02:52:38,662] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:53:07,427] INFO : Step 16000/500000 rank0 | Train Loss: 0.5156 | Train Loss Unweighted: 0.1593 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1359 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0116 | Regression_eta: 0.0001 | Regression_pt: 0.0116 | Regression_sin_phi: 0.0000 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0003 | Classification_binary: 0.1359 | Regression_cos_phi: 0.1014 | Regression_energy: 0.1086 | Regression_eta: 0.0376 | Regression_pt: 0.1116 | Regression_sin_phi: 0.0203 | LR: 3.33e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4638s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0366 | max_loss: 0.0026 | max_backward: 0.0380 | max_optimizer_step: 0.0591 | max_tensorboard_logging: 0.1313 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3092.2 | uss_MB: 1864.1 | child_rss_MB: 22764.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:53:07,429] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 02:53:07,496] INFO : Running validation on rank0 +rank0 [2026-11-15 02:53:07,496] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:53:07,496] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:53:07,620] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 02:53:07,727] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:53:07,829] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:53:07,897] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:53:07,961] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:53:08,026] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:53:08,092] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:53:08,157] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:53:08,269] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:53:08,376] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:53:08,485] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:53:08,550] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:53:08,614] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:53:08,678] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:53:08,743] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:53:08,808] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:53:08,903] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:53:08,908] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:53:08,911] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:53:08,913] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:53:08,915] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:53:08,916] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:53:08,917] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:53:08,919] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:53:08,921] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:53:08,922] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:53:08,923] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:53:08,924] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:53:08,926] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:53:08,928] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:53:08,930] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:53:08,931] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:53:10,321] INFO : VALIDATION | Step=16000/500000 | Train Loss=0.5156 | Valid Loss=9.3582 | Train Loss Unweighted=0.1593 | Valid Loss Unweighted=1.5123 | Stale=11 | ETA=948.5m +rank0 [2026-11-15 02:53:10,322] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:53:10,395] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:53:10,398] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:53:10,442] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:53:10,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:53:11,115] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:53:11,352] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:53:11,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:53:11,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:53:12,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:53:12,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:53:12,555] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:53:12,789] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:53:13,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:53:13,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:53:13,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:53:13,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:53:14,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:53:14,244] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:53:14,457] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:53:14,517] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:53:14,522] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:53:15,050] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:53:15,452] INFO : Plotted number of elements +rank0 [2026-11-15 02:53:16,706] INFO : Plotted jets +rank0 [2026-11-15 02:53:18,100] INFO : Plotted jet ratio +rank0 [2026-11-15 02:53:50,506] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 02:53:50,506] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:54:51,205] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 02:54:51,205] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:55:06,572] INFO : Step 17000/500000 rank0 | Train Loss: 1.1260 | Train Loss Unweighted: 0.3999 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.3402 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0298 | Regression_eta: 0.0002 | Regression_pt: 0.0295 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0029 | Classification_binary: 0.3402 | Regression_cos_phi: 0.0912 | Regression_energy: 0.2782 | Regression_eta: 0.0471 | Regression_pt: 0.2852 | Regression_sin_phi: 0.0813 | LR: 3.51e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4650s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0030 | max_backward: 0.0397 | max_optimizer_step: 0.0608 | max_tensorboard_logging: 0.1302 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3119.4 | uss_MB: 1891.3 | child_rss_MB: 22758.5 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:55:06,573] INFO : Running validation on rank0 +rank0 [2026-11-15 02:55:06,573] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:55:06,573] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:55:06,692] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 02:55:06,802] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:55:06,906] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:55:06,976] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:55:07,045] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:55:07,111] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:55:07,176] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:55:07,241] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:55:07,353] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:55:07,465] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:55:07,578] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:55:07,648] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:55:07,716] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:55:07,783] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:55:07,850] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:55:07,918] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:55:08,013] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:55:08,019] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:55:08,021] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:55:08,023] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:55:08,025] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:55:08,025] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:55:08,027] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:55:08,029] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:55:08,032] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:55:08,032] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:55:08,033] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:55:08,036] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:55:08,037] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:55:08,039] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:55:08,041] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:55:08,043] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:55:09,430] INFO : VALIDATION | Step=17000/500000 | Train Loss=1.1260 | Valid Loss=15.6847 | Train Loss Unweighted=0.3999 | Valid Loss Unweighted=1.5840 | Stale=12 | ETA=947.3m +rank0 [2026-11-15 02:55:09,430] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:55:09,505] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:55:09,508] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:55:09,551] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:55:09,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:55:10,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:55:10,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:55:10,714] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:55:10,953] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:55:11,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:55:11,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:55:11,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:55:11,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:55:12,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:55:12,377] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:55:12,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:55:12,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:55:13,111] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:55:13,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:55:13,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:55:13,616] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:55:13,622] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:55:14,156] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:55:14,559] INFO : Plotted number of elements +rank0 [2026-11-15 02:55:15,792] INFO : Plotted jets +rank0 [2026-11-15 02:55:17,160] INFO : Plotted jet ratio +rank0 [2026-11-15 02:56:03,082] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 02:56:03,082] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:57:03,566] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 02:57:03,566] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:57:05,668] INFO : Step 18000/500000 rank0 | Train Loss: 0.6758 | Train Loss Unweighted: 0.2777 | Train Loss Components: Classification: 0.0010 | Classification_binary: 0.2563 | Regression_cos_phi: 0.0000 | Regression_energy: 0.0101 | Regression_eta: 0.0001 | Regression_pt: 0.0101 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0807 | Classification_binary: 0.2563 | Regression_cos_phi: 0.0295 | Regression_energy: 0.0947 | Regression_eta: 0.0267 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0907 | LR: 3.69e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4624s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0394 | max_optimizer_step: 0.0612 | max_tensorboard_logging: 0.1280 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3142.9 | uss_MB: 1914.8 | child_rss_MB: 22766.1 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 02:57:05,670] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 02:57:05,735] INFO : Running validation on rank0 +rank0 [2026-11-15 02:57:05,735] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:57:05,736] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:57:05,854] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 02:57:05,960] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:57:06,061] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:57:06,128] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:57:06,194] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:57:06,261] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:57:06,328] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:57:06,396] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:57:06,509] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:57:06,620] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:57:06,730] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:57:06,797] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:57:06,862] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:57:06,926] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:57:06,991] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:57:07,057] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:57:07,152] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:57:07,158] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:57:07,160] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:57:07,161] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:57:07,164] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:57:07,165] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:57:07,167] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:57:07,168] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:57:07,170] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:57:07,171] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:57:07,173] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:57:07,175] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:57:07,175] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:57:07,177] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:57:07,179] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:57:07,181] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:57:08,581] INFO : VALIDATION | Step=18000/500000 | Train Loss=0.6758 | Valid Loss=15.0661 | Train Loss Unweighted=0.2777 | Valid Loss Unweighted=1.6344 | Stale=13 | ETA=946.0m +rank0 [2026-11-15 02:57:08,581] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:57:08,655] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:57:08,659] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:57:08,703] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:57:09,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:57:09,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:57:09,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:57:09,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:57:10,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:57:10,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:57:10,594] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:57:10,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:57:11,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:57:11,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:57:11,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:57:11,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:57:12,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:57:12,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:57:12,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:57:12,751] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:57:12,808] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:57:12,813] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:57:13,343] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:57:13,731] INFO : Plotted number of elements +rank0 [2026-11-15 02:57:14,973] INFO : Plotted jets +rank0 [2026-11-15 02:57:16,322] INFO : Plotted jet ratio +rank0 [2026-11-15 02:58:15,637] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 02:58:15,637] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:59:04,611] INFO : Step 19000/500000 rank0 | Train Loss: 1.4326 | Train Loss Unweighted: 0.7073 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.6686 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0179 | Regression_eta: 0.0008 | Regression_pt: 0.0193 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.6686 | Regression_cos_phi: 0.0615 | Regression_energy: 0.1675 | Regression_eta: 0.2139 | Regression_pt: 0.1864 | Regression_sin_phi: 0.0994 | LR: 3.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4547s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0383 | max_loss: 0.0026 | max_backward: 0.0389 | max_optimizer_step: 0.0595 | max_tensorboard_logging: 0.1271 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3169.0 | uss_MB: 1940.9 | child_rss_MB: 22764.0 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:59:04,612] INFO : Running validation on rank0 +rank0 [2026-11-15 02:59:04,612] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:59:04,612] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:59:04,731] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 02:59:04,834] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:59:04,933] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:59:05,002] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:59:05,067] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:59:05,131] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:59:05,196] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:59:05,264] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:59:05,376] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:59:05,479] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:59:05,581] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:59:05,647] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:59:05,713] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:59:05,778] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:59:05,845] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:59:05,908] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:59:06,003] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:59:06,007] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:59:06,009] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:59:06,013] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:59:06,015] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:59:06,015] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:59:06,017] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:59:06,019] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:59:06,020] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:59:06,021] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:59:06,023] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:59:06,025] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:59:06,026] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:59:06,027] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:59:06,028] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:59:06,032] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:59:07,420] INFO : VALIDATION | Step=19000/500000 | Train Loss=1.4326 | Valid Loss=13.0207 | Train Loss Unweighted=0.7073 | Valid Loss Unweighted=1.6752 | Stale=14 | ETA=944.5m +rank0 [2026-11-15 02:59:07,420] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:59:07,492] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:59:07,497] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:59:07,541] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:59:07,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:59:08,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:59:08,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:59:08,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:59:08,928] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:59:09,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:59:09,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:59:09,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:59:09,859] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:59:10,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:59:10,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:59:10,574] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:59:10,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:59:11,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:59:11,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:59:11,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:59:11,552] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:59:11,558] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:59:12,094] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:59:12,493] INFO : Plotted number of elements +rank0 [2026-11-15 02:59:13,736] INFO : Plotted jets +rank0 [2026-11-15 02:59:15,106] INFO : Plotted jet ratio +rank0 [2026-11-15 02:59:27,006] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 02:59:27,007] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:00:27,217] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 03:00:27,217] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:01:02,944] INFO : Step 20000/500000 rank0 | Train Loss: 0.6748 | Train Loss Unweighted: 0.1262 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0824 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0215 | Regression_eta: 0.0001 | Regression_pt: 0.0220 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0012 | Classification_binary: 0.0824 | Regression_cos_phi: 0.0560 | Regression_energy: 0.2009 | Regression_eta: 0.0214 | Regression_pt: 0.2129 | Regression_sin_phi: 0.1000 | LR: 4.07e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4618s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0026 | max_backward: 0.0387 | max_optimizer_step: 0.0587 | max_tensorboard_logging: 0.1305 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3123.2 | uss_MB: 1895.1 | child_rss_MB: 22766.0 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:01:02,946] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 03:01:03,018] INFO : Running validation on rank0 +rank0 [2026-11-15 03:01:03,018] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:01:03,018] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:01:03,144] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 03:01:03,255] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:01:03,357] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:01:03,425] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:01:03,490] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:01:03,553] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:01:03,618] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:01:03,683] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:01:03,793] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:01:03,906] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:01:04,017] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:01:04,089] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:01:04,156] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:01:04,222] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:01:04,287] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:01:04,353] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:01:04,451] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:01:04,457] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:01:04,459] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:01:04,461] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:01:04,463] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:01:04,465] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:01:04,466] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:01:04,468] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:01:04,470] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:01:04,470] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:01:04,472] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:01:04,474] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:01:04,475] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:01:04,476] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:01:04,479] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:01:04,481] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:01:05,871] INFO : VALIDATION | Step=20000/500000 | Train Loss=0.6748 | Valid Loss=13.2424 | Train Loss Unweighted=0.1262 | Valid Loss Unweighted=1.7804 | Stale=15 | ETA=942.8m +rank0 [2026-11-15 03:01:05,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:01:05,943] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:01:05,946] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:01:05,990] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:01:06,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:01:06,661] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:01:06,899] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:01:07,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:01:07,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:01:07,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:01:07,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:01:08,105] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:01:08,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:01:08,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:01:08,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:01:09,053] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:01:09,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:01:09,534] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:01:09,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:01:09,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:01:10,036] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:01:10,042] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:01:10,568] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:01:10,967] INFO : Plotted number of elements +rank0 [2026-11-15 03:01:12,216] INFO : Plotted jets +rank0 [2026-11-15 03:01:13,571] INFO : Plotted jet ratio +rank0 [2026-11-15 03:01:38,839] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 03:01:38,839] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:02:39,020] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 03:02:39,020] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:03:01,465] INFO : Step 21000/500000 rank0 | Train Loss: 1.0997 | Train Loss Unweighted: 0.8422 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.8260 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0081 | Regression_eta: 0.0001 | Regression_pt: 0.0078 | Regression_sin_phi: 0.0000 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0013 | Classification_binary: 0.8260 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0761 | Regression_eta: 0.0296 | Regression_pt: 0.0752 | Regression_sin_phi: 0.0301 | LR: 4.28e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4643s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0385 | max_optimizer_step: 0.0602 | max_tensorboard_logging: 0.1324 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3215.8 | uss_MB: 1987.7 | child_rss_MB: 22766.2 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:03:01,467] INFO : Running validation on rank0 +rank0 [2026-11-15 03:03:01,467] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:03:01,467] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:03:01,592] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 03:03:01,698] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:03:01,798] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:03:01,866] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:03:01,931] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:03:01,994] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:03:02,059] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:03:02,124] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:03:02,234] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:03:02,335] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:03:02,436] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:03:02,502] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:03:02,566] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:03:02,631] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:03:02,696] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:03:02,761] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:03:02,861] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:03:02,867] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:03:02,869] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:03:02,871] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:03:02,873] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:03:02,875] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:03:02,876] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:03:02,876] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:03:02,879] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:03:02,880] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:03:02,882] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:03:02,884] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:03:02,885] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:03:02,886] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:03:02,888] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:03:02,890] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:03:04,283] INFO : VALIDATION | Step=21000/500000 | Train Loss=1.0997 | Valid Loss=16.9017 | Train Loss Unweighted=0.8422 | Valid Loss Unweighted=1.9002 | Stale=16 | ETA=941.0m +rank0 [2026-11-15 03:03:04,284] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:03:04,357] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:03:04,360] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:03:04,404] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:03:04,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:03:05,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:03:05,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:03:05,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:03:05,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:03:06,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:03:06,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:03:06,523] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:03:06,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:03:06,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:03:07,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:03:07,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:03:07,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:03:07,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:03:08,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:03:08,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:03:08,496] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:03:08,502] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:03:09,027] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:03:09,456] INFO : Plotted number of elements +rank0 [2026-11-15 03:03:10,679] INFO : Plotted jets +rank0 [2026-11-15 03:03:12,041] INFO : Plotted jet ratio +rank0 [2026-11-15 03:03:50,583] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 03:03:50,584] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:04:50,832] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 03:04:50,832] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:05:00,006] INFO : Step 22000/500000 rank0 | Train Loss: 0.4749 | Train Loss Unweighted: 0.0815 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0578 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0121 | Regression_eta: 0.0003 | Regression_pt: 0.0112 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0006 | Classification_binary: 0.0578 | Regression_cos_phi: 0.0744 | Regression_energy: 0.1128 | Regression_eta: 0.0669 | Regression_pt: 0.1082 | Regression_sin_phi: 0.0543 | LR: 4.50e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4633s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0385 | max_optimizer_step: 0.0594 | max_tensorboard_logging: 0.1314 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3249.7 | uss_MB: 2021.6 | child_rss_MB: 22766.2 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 12.0 +rank0 [2026-11-15 03:05:00,007] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 03:05:00,075] INFO : Running validation on rank0 +rank0 [2026-11-15 03:05:00,075] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:05:00,075] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:05:00,195] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 03:05:00,300] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:05:00,401] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:05:00,468] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:05:00,532] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:05:00,596] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:05:00,661] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:05:00,726] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:05:00,838] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:05:00,946] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:05:01,054] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:05:01,122] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:05:01,187] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:05:01,250] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:05:01,315] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:05:01,380] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:05:01,478] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:05:01,484] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:05:01,487] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:05:01,489] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:05:01,491] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:05:01,493] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:05:01,493] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:05:01,494] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:05:01,497] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:05:01,498] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:05:01,500] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:05:01,502] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:05:01,502] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:05:01,504] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:05:01,506] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:05:01,508] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:05:02,885] INFO : VALIDATION | Step=22000/500000 | Train Loss=0.4749 | Valid Loss=15.5750 | Train Loss Unweighted=0.0815 | Valid Loss Unweighted=1.9360 | Stale=17 | ETA=939.3m +rank0 [2026-11-15 03:05:02,885] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:05:02,966] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:05:02,969] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:05:03,012] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:05:03,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:05:03,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:05:03,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:05:04,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:05:04,438] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:05:04,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:05:04,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:05:05,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:05:05,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:05:05,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:05:05,881] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:05:06,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:05:06,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:05:06,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:05:06,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:05:07,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:05:07,110] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:05:07,115] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:05:07,648] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:05:08,039] INFO : Plotted number of elements +rank0 [2026-11-15 03:05:09,297] INFO : Plotted jets +rank0 [2026-11-15 03:05:10,701] INFO : Plotted jet ratio +rank0 [2026-11-15 03:06:02,714] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 03:06:02,714] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:06:58,885] INFO : Step 23000/500000 rank0 | Train Loss: 1.1171 | Train Loss Unweighted: 0.3516 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.2971 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0262 | Regression_eta: 0.0002 | Regression_pt: 0.0277 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0010 | Classification_binary: 0.2971 | Regression_cos_phi: 0.1174 | Regression_energy: 0.2453 | Regression_eta: 0.0555 | Regression_pt: 0.2674 | Regression_sin_phi: 0.1334 | LR: 4.73e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4579s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0390 | max_optimizer_step: 0.0590 | max_tensorboard_logging: 0.1263 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3261.5 | uss_MB: 2033.4 | child_rss_MB: 22765.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:06:58,887] INFO : Running validation on rank0 +rank0 [2026-11-15 03:06:58,887] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:06:58,887] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:06:59,011] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 03:06:59,123] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:06:59,228] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:06:59,298] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:06:59,365] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:06:59,431] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:06:59,499] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:06:59,565] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:06:59,677] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:06:59,789] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:06:59,901] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:06:59,968] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:07:00,035] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:07:00,103] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:07:00,168] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:07:00,233] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:07:00,331] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:07:00,336] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:07:00,338] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:07:00,341] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:07:00,343] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:07:00,343] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:07:00,345] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:07:00,347] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:07:00,349] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:07:00,349] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:07:00,351] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:07:00,353] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:07:00,355] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:07:00,357] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:07:00,359] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:07:00,360] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:07:01,748] INFO : VALIDATION | Step=23000/500000 | Train Loss=1.1171 | Valid Loss=18.5581 | Train Loss Unweighted=0.3516 | Valid Loss Unweighted=2.0273 | Stale=18 | ETA=937.7m +rank0 [2026-11-15 03:07:01,748] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:07:01,820] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:07:01,823] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:07:01,867] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:07:02,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:07:02,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:07:02,810] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:07:03,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:07:03,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:07:03,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:07:03,759] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:07:04,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:07:04,245] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:07:04,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:07:04,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:07:04,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:07:05,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:07:05,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:07:05,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:07:05,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:07:05,943] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:07:05,948] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:07:06,477] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:07:06,865] INFO : Plotted number of elements +rank0 [2026-11-15 03:07:08,110] INFO : Plotted jets +rank0 [2026-11-15 03:07:09,458] INFO : Plotted jet ratio +rank0 [2026-11-15 03:07:14,314] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 03:07:14,314] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:08:14,715] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 03:08:14,715] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:08:57,660] INFO : Step 24000/500000 rank0 | Train Loss: 1.0532 | Train Loss Unweighted: 0.4812 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.4395 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0205 | Regression_eta: 0.0003 | Regression_pt: 0.0206 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0015 | Classification_binary: 0.4395 | Regression_cos_phi: 0.0641 | Regression_energy: 0.1920 | Regression_eta: 0.0676 | Regression_pt: 0.1994 | Regression_sin_phi: 0.0889 | LR: 4.97e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4618s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0388 | max_optimizer_step: 0.0596 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3299.7 | uss_MB: 2071.6 | child_rss_MB: 22764.4 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:08:57,661] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 03:08:57,727] INFO : Running validation on rank0 +rank0 [2026-11-15 03:08:57,727] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:08:57,727] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:08:57,844] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 03:08:57,949] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:08:58,049] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:08:58,118] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:08:58,184] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:08:58,250] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:08:58,317] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:08:58,384] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:08:58,498] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:08:58,600] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:08:58,701] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:08:58,767] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:08:58,831] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:08:58,894] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:08:58,958] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:08:59,023] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:08:59,121] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:08:59,127] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:08:59,129] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:08:59,130] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:08:59,133] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:08:59,135] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:08:59,136] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:08:59,138] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:08:59,140] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:08:59,140] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:08:59,142] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:08:59,144] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:08:59,145] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:08:59,146] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:08:59,148] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:08:59,151] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:09:00,540] INFO : VALIDATION | Step=24000/500000 | Train Loss=1.0532 | Valid Loss=16.8185 | Train Loss Unweighted=0.4812 | Valid Loss Unweighted=2.0502 | Stale=19 | ETA=936.0m +rank0 [2026-11-15 03:09:00,541] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:09:00,615] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:09:00,619] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:09:00,664] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:09:01,133] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:09:01,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:09:01,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:09:01,874] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:09:02,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:09:02,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:09:02,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:09:02,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:09:03,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:09:03,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:09:03,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:09:03,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:09:04,077] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:09:04,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:09:04,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:09:04,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:09:04,828] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:09:04,834] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:09:05,368] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:09:05,762] INFO : Plotted number of elements +rank0 [2026-11-15 03:09:07,014] INFO : Plotted jets +rank0 [2026-11-15 03:09:08,377] INFO : Plotted jet ratio +rank0 [2026-11-15 03:09:26,727] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 03:09:26,727] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:10:27,705] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 03:10:27,705] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:10:57,509] INFO : Step 25000/500000 rank0 | Train Loss: 1.9763 | Train Loss Unweighted: 0.5359 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.4049 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0651 | Regression_eta: 0.0004 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.4049 | Regression_cos_phi: 0.1668 | Regression_energy: 0.6085 | Regression_eta: 0.0944 | Regression_pt: 0.6302 | Regression_sin_phi: 0.0715 | LR: 5.22e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5551s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0589 | max_tensorboard_logging: 0.2224 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3325.9 | uss_MB: 2097.8 | child_rss_MB: 22762.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:10:57,511] INFO : Running validation on rank0 +rank0 [2026-11-15 03:10:57,511] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:10:57,511] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:10:57,629] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 03:10:57,733] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:10:57,835] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:10:57,904] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:10:57,971] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:10:58,037] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:10:58,104] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:10:58,170] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:10:58,281] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:10:58,385] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:10:58,488] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:10:58,555] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:10:58,620] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:10:58,684] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:10:58,748] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:10:58,812] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:10:58,910] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:10:58,916] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:10:58,918] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:10:58,920] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:10:58,922] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:10:58,924] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:10:58,926] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:10:58,928] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:10:58,930] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:10:58,931] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:10:58,933] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:10:58,935] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:10:58,936] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:10:58,938] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:10:58,940] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:10:58,941] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:11:00,328] INFO : VALIDATION | Step=25000/500000 | Train Loss=1.9763 | Valid Loss=13.1204 | Train Loss Unweighted=0.5359 | Valid Loss Unweighted=2.2639 | Stale=20 | ETA=934.6m +rank0 [2026-11-15 03:11:00,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:11:00,448] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:11:00,454] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:11:00,495] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:11:00,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:11:01,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:11:01,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:11:01,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:11:01,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:11:02,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:11:02,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:11:02,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:11:02,837] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:11:03,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:11:03,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:11:03,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:11:03,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:11:04,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:11:04,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:11:04,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:11:04,551] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:11:04,556] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:11:05,088] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:11:05,476] INFO : Plotted number of elements +rank0 [2026-11-15 03:11:06,718] INFO : Plotted jets +rank0 [2026-11-15 03:11:08,059] INFO : Plotted jet ratio +rank0 [2026-11-15 03:11:39,569] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 03:11:39,569] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:12:40,252] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 03:12:40,253] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:12:56,666] INFO : Step 26000/500000 rank0 | Train Loss: 0.5243 | Train Loss Unweighted: 0.0864 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0558 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0154 | Regression_eta: 0.0001 | Regression_pt: 0.0149 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0010 | Classification_binary: 0.0558 | Regression_cos_phi: 0.0874 | Regression_energy: 0.1444 | Regression_eta: 0.0229 | Regression_pt: 0.1435 | Regression_sin_phi: 0.0693 | LR: 5.47e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4727s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0030 | max_backward: 0.0403 | max_optimizer_step: 0.0609 | max_tensorboard_logging: 0.1300 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3338.3 | uss_MB: 2110.1 | child_rss_MB: 22766.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:12:56,668] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 03:12:56,736] INFO : Running validation on rank0 +rank0 [2026-11-15 03:12:56,736] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:12:56,736] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:12:56,857] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 03:12:56,963] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:12:57,063] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:12:57,130] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:12:57,195] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:12:57,259] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:12:57,325] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:12:57,391] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:12:57,504] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:12:57,608] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:12:57,715] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:12:57,782] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:12:57,848] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:12:57,915] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:12:57,982] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:12:58,049] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:12:58,150] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:12:58,156] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:12:58,158] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:12:58,160] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:12:58,162] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:12:58,164] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:12:58,165] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:12:58,167] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:12:58,168] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:12:58,169] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:12:58,171] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:12:58,173] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:12:58,174] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:12:58,177] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:12:58,179] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:12:58,181] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:12:59,588] INFO : VALIDATION | Step=26000/500000 | Train Loss=0.5243 | Valid Loss=18.7163 | Train Loss Unweighted=0.0864 | Valid Loss Unweighted=2.3894 | Stale=21 | ETA=933.0m +rank0 [2026-11-15 03:12:59,589] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:12:59,662] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:12:59,666] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:12:59,712] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:13:00,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:13:00,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:13:00,650] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:13:00,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:13:01,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:13:01,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:13:01,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:13:01,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:13:02,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:13:02,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:13:02,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:13:02,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:13:03,053] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:13:03,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:13:03,526] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:13:03,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:13:03,800] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:13:03,805] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:13:04,344] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:13:04,748] INFO : Plotted number of elements +rank0 [2026-11-15 03:13:05,986] INFO : Plotted jets +rank0 [2026-11-15 03:13:07,377] INFO : Plotted jet ratio +rank0 [2026-11-15 03:13:52,392] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 03:13:52,393] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:14:52,880] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 03:14:52,881] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:14:55,839] INFO : Step 27000/500000 rank0 | Train Loss: 0.5833 | Train Loss Unweighted: 0.1423 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1182 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0119 | Regression_eta: 0.0002 | Regression_pt: 0.0118 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.1182 | Regression_cos_phi: 0.1112 | Regression_energy: 0.1113 | Regression_eta: 0.0491 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0794 | LR: 5.74e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4578s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0382 | max_optimizer_step: 0.0587 | max_tensorboard_logging: 0.1287 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3372.7 | uss_MB: 2144.6 | child_rss_MB: 22766.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:14:55,840] INFO : Running validation on rank0 +rank0 [2026-11-15 03:14:55,841] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:14:55,841] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:14:55,964] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 03:14:56,074] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:14:56,175] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:14:56,245] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:14:56,311] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:14:56,378] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:14:56,446] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:14:56,513] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:14:56,629] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:14:56,734] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:14:56,835] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:14:56,901] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:14:56,966] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:14:57,030] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:14:57,094] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:14:57,158] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:14:57,257] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:14:57,263] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:14:57,265] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:14:57,267] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:14:57,269] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:14:57,271] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:14:57,272] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:14:57,274] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:14:57,275] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:14:57,276] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:14:57,278] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:14:57,280] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:14:57,280] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:14:57,282] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:14:57,284] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:14:57,286] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:14:58,689] INFO : VALIDATION | Step=27000/500000 | Train Loss=0.5833 | Valid Loss=18.1313 | Train Loss Unweighted=0.1423 | Valid Loss Unweighted=2.3444 | Stale=22 | ETA=931.3m +rank0 [2026-11-15 03:14:58,689] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:14:58,763] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:14:58,767] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:14:58,812] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:14:59,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:14:59,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:14:59,766] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:15:00,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:15:00,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:15:00,473] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:15:00,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:15:00,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:15:01,188] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:15:01,437] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:15:01,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:15:01,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:15:02,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:15:02,383] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:15:02,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:15:02,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:15:02,881] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:15:02,886] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:15:03,425] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:15:03,816] INFO : Plotted number of elements +rank0 [2026-11-15 03:15:05,065] INFO : Plotted jets +rank0 [2026-11-15 03:15:06,434] INFO : Plotted jet ratio +rank0 [2026-11-15 03:16:04,679] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 03:16:04,679] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:16:54,628] INFO : Step 28000/500000 rank0 | Train Loss: 0.8734 | Train Loss Unweighted: 0.1075 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.0565 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0249 | Regression_eta: 0.0001 | Regression_pt: 0.0254 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0218 | Classification_binary: 0.0565 | Regression_cos_phi: 0.1361 | Regression_energy: 0.2328 | Regression_eta: 0.0252 | Regression_pt: 0.2454 | Regression_sin_phi: 0.1556 | LR: 6.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4700s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0026 | max_backward: 0.0385 | max_optimizer_step: 0.0605 | max_tensorboard_logging: 0.1281 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3395.8 | uss_MB: 2167.7 | child_rss_MB: 22764.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:16:54,630] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 03:16:54,698] INFO : Running validation on rank0 +rank0 [2026-11-15 03:16:54,698] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:16:54,698] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:16:54,816] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 03:16:54,923] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:16:55,024] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:16:55,093] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:16:55,160] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:16:55,226] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:16:55,295] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:16:55,363] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:16:55,476] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:16:55,580] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:16:55,683] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:16:55,750] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:16:55,816] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:16:55,881] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:16:55,947] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:16:56,011] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:16:56,110] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:16:56,116] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:16:56,118] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:16:56,120] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:16:56,122] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:16:56,124] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:16:56,125] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:16:56,125] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:16:56,128] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:16:56,129] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:16:56,130] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:16:56,133] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:16:56,133] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:16:56,135] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:16:56,137] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:16:56,139] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:16:57,546] INFO : VALIDATION | Step=28000/500000 | Train Loss=0.8734 | Valid Loss=16.8242 | Train Loss Unweighted=0.1075 | Valid Loss Unweighted=2.3713 | Stale=23 | ETA=929.6m +rank0 [2026-11-15 03:16:57,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:16:57,619] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:16:57,625] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:16:57,675] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:16:58,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:16:58,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:16:58,646] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:16:58,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:16:59,115] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:16:59,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:16:59,586] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:16:59,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:17:00,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:17:00,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:17:00,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:17:00,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:17:01,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:17:01,249] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:17:01,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:17:01,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:17:01,770] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:17:01,779] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:17:02,311] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:17:02,700] INFO : Plotted number of elements +rank0 [2026-11-15 03:17:03,933] INFO : Plotted jets +rank0 [2026-11-15 03:17:05,285] INFO : Plotted jet ratio +rank0 [2026-11-15 03:17:16,306] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 03:17:16,306] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:18:16,338] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 03:18:16,338] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:18:52,829] INFO : Step 29000/500000 rank0 | Train Loss: 1.0738 | Train Loss Unweighted: 0.1216 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0371 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0414 | Regression_eta: 0.0003 | Regression_pt: 0.0425 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0002 | Classification_binary: 0.0371 | Regression_cos_phi: 0.0910 | Regression_energy: 0.3872 | Regression_eta: 0.0666 | Regression_pt: 0.4104 | Regression_sin_phi: 0.0812 | LR: 6.29e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4557s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0589 | max_tensorboard_logging: 0.1267 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3413.2 | uss_MB: 2185.1 | child_rss_MB: 22766.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:18:52,831] INFO : Running validation on rank0 +rank0 [2026-11-15 03:18:52,831] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:18:52,831] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:18:52,949] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 03:18:53,054] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:18:53,153] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:18:53,221] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:18:53,287] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:18:53,353] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:18:53,420] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:18:53,487] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:18:53,601] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:18:53,711] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:18:53,819] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:18:53,885] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:18:53,949] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:18:54,013] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:18:54,078] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:18:54,142] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:18:54,240] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:18:54,246] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:18:54,248] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:18:54,250] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:18:54,252] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:18:54,254] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:18:54,255] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:18:54,257] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:18:54,258] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:18:54,259] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:18:54,261] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:18:54,263] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:18:54,265] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:18:54,265] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:18:54,267] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:18:54,271] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:18:55,659] INFO : VALIDATION | Step=29000/500000 | Train Loss=1.0738 | Valid Loss=14.2513 | Train Loss Unweighted=0.1216 | Valid Loss Unweighted=2.5250 | Stale=24 | ETA=927.6m +rank0 [2026-11-15 03:18:55,659] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:18:55,731] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:18:55,735] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:18:55,780] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:18:56,251] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:18:56,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:18:56,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:18:56,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:18:57,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:18:57,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:18:57,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:18:57,910] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:18:58,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:18:58,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:18:58,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:18:58,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:18:59,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:18:59,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:18:59,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:18:59,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:18:59,856] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:18:59,861] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:19:00,403] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:19:00,791] INFO : Plotted number of elements +rank0 [2026-11-15 03:19:02,040] INFO : Plotted jets +rank0 [2026-11-15 03:19:03,390] INFO : Plotted jet ratio +rank0 [2026-11-15 03:19:27,670] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 03:19:27,670] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:20:27,946] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 03:20:27,947] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:20:51,291] INFO : Step 30000/500000 rank0 | Train Loss: 0.4130 | Train Loss Unweighted: 0.0748 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0491 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0125 | Regression_eta: 0.0002 | Regression_pt: 0.0129 | Regression_sin_phi: 0.0000 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0000 | Classification_binary: 0.0491 | Regression_cos_phi: 0.0444 | Regression_energy: 0.1172 | Regression_eta: 0.0425 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0353 | LR: 6.58e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4973s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0354 | max_loss: 0.0027 | max_backward: 0.0391 | max_optimizer_step: 0.0745 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3436.3 | uss_MB: 2208.2 | child_rss_MB: 22766.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:20:51,293] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 03:20:51,361] INFO : Running validation on rank0 +rank0 [2026-11-15 03:20:51,362] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:20:51,362] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:20:51,478] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 03:20:51,584] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:20:51,687] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:20:51,755] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:20:51,821] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:20:51,887] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:20:51,954] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:20:52,020] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:20:52,136] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:20:52,239] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:20:52,342] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:20:52,410] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:20:52,476] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:20:52,544] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:20:52,611] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:20:52,678] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:20:52,779] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:20:52,785] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:20:52,787] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:20:52,789] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:20:52,791] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:20:52,793] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:20:52,794] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:20:52,796] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:20:52,797] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:20:52,798] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:20:52,800] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:20:52,802] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:20:52,803] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:20:52,804] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:20:52,806] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:20:52,808] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:20:54,197] INFO : VALIDATION | Step=30000/500000 | Train Loss=0.4130 | Valid Loss=18.9530 | Train Loss Unweighted=0.0748 | Valid Loss Unweighted=2.6810 | Stale=25 | ETA=925.7m +rank0 [2026-11-15 03:20:54,197] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:20:54,271] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:20:54,274] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:54,319] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:20:54,768] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:20:55,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:20:55,251] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:20:55,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:20:55,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:20:55,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:20:56,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:20:56,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:20:56,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:20:56,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:20:57,161] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:20:57,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:20:57,645] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:20:57,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:20:58,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:20:58,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:20:58,391] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:20:58,396] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:58,930] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:59,331] INFO : Plotted number of elements +rank0 [2026-11-15 03:21:00,558] INFO : Plotted jets +rank0 [2026-11-15 03:21:01,922] INFO : Plotted jet ratio +rank0 [2026-11-15 03:21:39,529] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 03:21:39,529] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:22:39,989] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 03:22:39,989] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:22:50,015] INFO : Step 31000/500000 rank0 | Train Loss: 0.7005 | Train Loss Unweighted: 0.1567 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1090 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0237 | Regression_eta: 0.0001 | Regression_pt: 0.0237 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.1090 | Regression_cos_phi: 0.0381 | Regression_energy: 0.2215 | Regression_eta: 0.0339 | Regression_pt: 0.2294 | Regression_sin_phi: 0.0686 | LR: 6.88e-05 | DataLoad Time: 0.0004s | Model Forward Time: 0.4822s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0379 | max_loss: 0.0031 | max_backward: 0.0401 | max_optimizer_step: 0.0623 | max_tensorboard_logging: 0.1340 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3392.6 | uss_MB: 2164.5 | child_rss_MB: 22766.2 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:22:50,016] INFO : Running validation on rank0 +rank0 [2026-11-15 03:22:50,016] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:22:50,016] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:22:50,137] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 03:22:50,245] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:22:50,346] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:22:50,416] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:22:50,482] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:22:50,548] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:22:50,614] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:22:50,680] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:22:50,792] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:22:50,903] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:22:51,015] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:22:51,084] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:22:51,152] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:22:51,217] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:22:51,284] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:22:51,352] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:22:51,456] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:22:51,462] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:22:51,464] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:22:51,466] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:22:51,468] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:22:51,470] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:22:51,471] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:22:51,473] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:22:51,473] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:22:51,475] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:22:51,477] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:22:51,479] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:22:51,481] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:22:51,483] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:22:51,483] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:22:51,487] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:22:52,912] INFO : VALIDATION | Step=31000/500000 | Train Loss=0.7005 | Valid Loss=19.4268 | Train Loss Unweighted=0.1567 | Valid Loss Unweighted=2.6205 | Stale=26 | ETA=923.9m +rank0 [2026-11-15 03:22:52,913] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:22:52,987] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:22:52,991] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:53,037] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:22:53,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:22:53,729] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:22:53,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:22:54,222] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:22:54,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:22:54,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:22:54,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:22:55,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:22:55,420] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:22:55,659] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:22:55,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:22:56,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:22:56,387] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:22:56,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:22:56,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:22:57,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:22:57,140] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:22:57,146] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:57,680] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:58,084] INFO : Plotted number of elements +rank0 [2026-11-15 03:22:59,310] INFO : Plotted jets +rank0 [2026-11-15 03:23:00,681] INFO : Plotted jet ratio +rank0 [2026-11-15 03:23:51,821] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-15 03:23:51,821] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:24:49,037] INFO : Step 32000/500000 rank0 | Train Loss: 0.9360 | Train Loss Unweighted: 0.1525 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1000 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0273 | Regression_eta: 0.0006 | Regression_pt: 0.0244 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0002 | Classification_binary: 0.1000 | Regression_cos_phi: 0.0914 | Regression_energy: 0.2549 | Regression_eta: 0.1608 | Regression_pt: 0.2360 | Regression_sin_phi: 0.0927 | LR: 7.19e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4730s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0028 | max_backward: 0.0395 | max_optimizer_step: 0.0628 | max_tensorboard_logging: 0.1288 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3492.3 | uss_MB: 2264.2 | child_rss_MB: 22765.4 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:24:49,039] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-32000.pth +rank0 [2026-11-15 03:24:49,108] INFO : Running validation on rank0 +rank0 [2026-11-15 03:24:49,108] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:24:49,108] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:24:49,226] INFO : saved batch0_epoch32000.parquet +rank0 [2026-11-15 03:24:49,330] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:24:49,430] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:24:49,496] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:24:49,561] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:24:49,625] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:24:49,691] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:24:49,757] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:24:49,871] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:24:49,982] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:24:50,094] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:24:50,162] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:24:50,229] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:24:50,293] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:24:50,358] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:24:50,423] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:24:50,523] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:24:50,529] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:24:50,531] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:24:50,533] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:24:50,535] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:24:50,537] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:24:50,537] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:24:50,539] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:24:50,541] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:24:50,542] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:24:50,544] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:24:50,544] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:24:50,546] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:24:50,548] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:24:50,550] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:24:50,552] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:24:51,948] INFO : VALIDATION | Step=32000/500000 | Train Loss=0.9360 | Valid Loss=19.7769 | Train Loss Unweighted=0.1525 | Valid Loss Unweighted=2.7567 | Stale=27 | ETA=922.1m +rank0 [2026-11-15 03:24:51,949] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:24:52,022] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:24:52,027] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:52,074] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:24:52,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:24:52,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:24:53,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:24:53,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:24:53,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:24:53,737] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:24:53,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:24:54,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:24:54,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:24:54,690] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:24:54,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:24:55,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:24:55,409] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:24:55,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:24:55,878] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:24:56,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:24:56,159] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:24:56,164] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:56,715] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:57,111] INFO : Plotted number of elements +rank0 [2026-11-15 03:24:58,365] INFO : Plotted jets +rank0 [2026-11-15 03:24:59,730] INFO : Plotted jet ratio +rank0 [2026-11-15 03:25:03,716] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-15 03:25:03,716] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:26:04,244] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-15 03:26:04,245] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:26:48,005] INFO : Step 33000/500000 rank0 | Train Loss: 0.6567 | Train Loss Unweighted: 0.0826 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0527 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0146 | Regression_eta: 0.0003 | Regression_pt: 0.0147 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0007 | Classification_binary: 0.0527 | Regression_cos_phi: 0.1668 | Regression_energy: 0.1362 | Regression_eta: 0.0693 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0890 | LR: 7.51e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4899s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0394 | max_loss: 0.0030 | max_backward: 0.0408 | max_optimizer_step: 0.0650 | max_tensorboard_logging: 0.1280 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3515.3 | uss_MB: 2287.2 | child_rss_MB: 22763.7 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:26:48,006] INFO : Running validation on rank0 +rank0 [2026-11-15 03:26:48,006] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:26:48,006] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:26:48,125] INFO : saved batch0_epoch33000.parquet +rank0 [2026-11-15 03:26:48,230] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:26:48,329] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:26:48,399] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:26:48,466] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:26:48,532] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:26:48,598] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:26:48,663] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:26:48,776] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:26:48,882] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:26:48,988] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:26:49,056] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:26:49,123] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:26:49,189] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:26:49,255] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:26:49,319] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:26:49,421] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:26:49,427] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:26:49,429] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:26:49,431] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:26:49,433] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:26:49,435] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:26:49,436] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:26:49,438] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:26:49,440] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:26:49,440] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:26:49,442] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:26:49,444] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:26:49,446] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:26:49,448] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:26:49,450] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:26:49,452] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:26:50,848] INFO : VALIDATION | Step=33000/500000 | Train Loss=0.6567 | Valid Loss=16.3618 | Train Loss Unweighted=0.0826 | Valid Loss Unweighted=2.9069 | Stale=28 | ETA=920.3m +rank0 [2026-11-15 03:26:50,849] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:26:50,923] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:26:50,927] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:50,974] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:26:51,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:26:51,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:26:51,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:26:52,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:26:52,398] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:26:52,635] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:26:52,870] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:26:53,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:26:53,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:26:53,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:26:53,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:26:54,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:26:54,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:26:54,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:26:54,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:26:54,983] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:26:55,057] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:26:55,062] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:55,597] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:56,001] INFO : Plotted number of elements +rank0 [2026-11-15 03:26:57,234] INFO : Plotted jets +rank0 [2026-11-15 03:26:58,607] INFO : Plotted jet ratio +rank0 [2026-11-15 03:27:15,898] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-15 03:27:15,898] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:28:16,059] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-15 03:28:16,059] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:28:46,468] INFO : Step 34000/500000 rank0 | Train Loss: 1.1343 | Train Loss Unweighted: 0.3672 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.2947 | Regression_cos_phi: 0.0000 | Regression_energy: 0.0360 | Regression_eta: 0.0002 | Regression_pt: 0.0361 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.2947 | Regression_cos_phi: 0.0233 | Regression_energy: 0.3364 | Regression_eta: 0.0460 | Regression_pt: 0.3487 | Regression_sin_phi: 0.0850 | LR: 7.83e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4764s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0384 | max_optimizer_step: 0.0608 | max_tensorboard_logging: 0.1349 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3466.6 | uss_MB: 2238.5 | child_rss_MB: 22766.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:28:46,470] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-34000.pth +rank0 [2026-11-15 03:28:46,537] INFO : Running validation on rank0 +rank0 [2026-11-15 03:28:46,537] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:28:46,537] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:28:46,661] INFO : saved batch0_epoch34000.parquet +rank0 [2026-11-15 03:28:46,771] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:28:46,875] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:28:46,944] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:28:47,010] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:28:47,076] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:28:47,142] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:28:47,208] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:28:47,321] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:28:47,431] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:28:47,541] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:28:47,607] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:28:47,672] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:28:47,737] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:28:47,802] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:28:47,867] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:28:47,970] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:28:47,976] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:28:47,978] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:28:47,980] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:28:47,982] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:28:47,984] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:28:47,985] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:28:47,986] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:28:47,989] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:28:47,989] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:28:47,991] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:28:47,992] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:28:47,994] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:28:47,996] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:28:47,996] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:28:48,000] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:28:49,429] INFO : VALIDATION | Step=34000/500000 | Train Loss=1.1343 | Valid Loss=14.8704 | Train Loss Unweighted=0.3672 | Valid Loss Unweighted=2.8519 | Stale=29 | ETA=918.4m +rank0 [2026-11-15 03:28:49,429] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:28:49,505] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:28:49,508] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:49,559] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:28:50,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:28:50,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:28:50,544] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:28:50,789] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:28:51,028] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:28:51,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:28:51,510] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:28:51,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:28:51,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:28:52,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:28:52,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:28:52,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:28:52,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:28:53,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:28:53,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:28:53,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:28:53,712] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:28:53,718] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:54,245] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:54,646] INFO : Plotted number of elements +rank0 [2026-11-15 03:28:55,889] INFO : Plotted jets +rank0 [2026-11-15 03:28:57,247] INFO : Plotted jet ratio +rank0 [2026-11-15 03:29:27,798] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-15 03:29:27,799] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:30:28,618] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-15 03:30:28,618] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:30:45,701] INFO : Step 35000/500000 rank0 | Train Loss: 0.4871 | Train Loss Unweighted: 0.1599 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1391 | Regression_cos_phi: 0.0000 | Regression_energy: 0.0105 | Regression_eta: 0.0003 | Regression_pt: 0.0099 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0002 | Classification_binary: 0.1391 | Regression_cos_phi: 0.0314 | Regression_energy: 0.0985 | Regression_eta: 0.0742 | Regression_pt: 0.0953 | Regression_sin_phi: 0.0484 | LR: 8.16e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4638s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0601 | max_tensorboard_logging: 0.1293 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3567.4 | uss_MB: 2339.3 | child_rss_MB: 22766.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:30:45,703] INFO : Running validation on rank0 +rank0 [2026-11-15 03:30:45,703] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:30:45,703] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:30:45,823] INFO : saved batch0_epoch35000.parquet +rank0 [2026-11-15 03:30:45,927] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:30:46,028] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:30:46,098] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:30:46,164] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:30:46,231] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:30:46,299] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:30:46,368] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:30:46,483] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:30:46,596] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:30:46,706] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:30:46,773] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:30:46,839] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:30:46,902] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:30:46,967] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:30:47,032] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:30:47,135] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:30:47,141] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:30:47,143] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:30:47,145] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:30:47,147] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:30:47,149] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:30:47,150] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:30:47,151] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:30:47,153] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:30:47,155] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:30:47,156] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:30:47,158] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:30:47,159] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:30:47,161] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:30:47,163] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:30:47,165] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:30:48,579] INFO : VALIDATION | Step=35000/500000 | Train Loss=0.4871 | Valid Loss=19.2367 | Train Loss Unweighted=0.1599 | Valid Loss Unweighted=3.1031 | Stale=30 | ETA=916.6m +rank0 [2026-11-15 03:30:48,579] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:30:48,654] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:30:48,657] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:48,705] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:30:49,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:30:49,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:30:49,675] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:30:49,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:30:50,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:30:50,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:30:50,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:30:50,869] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:30:51,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:30:51,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:30:51,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:30:51,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:30:52,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:30:52,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:30:52,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:30:52,755] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:30:52,803] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:30:52,809] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:53,348] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:53,741] INFO : Plotted number of elements +rank0 [2026-11-15 03:30:54,983] INFO : Plotted jets +rank0 [2026-11-15 03:30:56,334] INFO : Plotted jet ratio +rank0 [2026-11-15 03:30:56,675] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 03:30:56,675] INFO : Training completed. Total time on device 0: 69.130min diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..c0301ac2cdb4729ac3ef17c367f7ee61909ddf45 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/train.log.0 @@ -0,0 +1,2763 @@ +rank0 [2026-11-15 02:21:45,906] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 02:21:45,906] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 02:21:45,907] INFO : Instantiating model +rank0 [2026-11-15 02:21:45,934] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 02:21:45,934] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 02:21:45,936] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 02:21:45,936] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 02:21:45,936] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 02:21:45,936] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 02:21:45,936] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,939] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 02:21:45,939] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,941] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 02:21:45,941] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,944] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 02:21:45,944] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,947] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 02:21:45,947] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,949] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 02:21:45,949] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,952] INFO : Backbone initialization took 0.02s +rank0 [2026-11-15 02:21:45,952] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 02:21:45,952] INFO : Initializing output DNNs +rank0 [2026-11-15 02:21:45,965] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,968] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:21:45,975] INFO : Output DNNs initialization took 0.02s +rank0 [2026-11-15 02:21:45,975] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 02:21:45,975] INFO : jagged_attention enabled=True +rank0 [2026-11-15 02:21:45,975] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 02:21:45,975] INFO : task_queries enabled=True +rank0 [2026-11-15 02:21:45,975] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 02:21:45,975] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 02:21:45,975] INFO : classification_query parameters: 256 +rank0 [2026-11-15 02:21:45,975] INFO : regression_query parameters: 256 +rank0 [2026-11-15 02:21:45,975] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 02:21:45,976] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 02:21:45,976] INFO : output_mode=elementwise +rank0 [2026-11-15 02:21:45,976] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 02:21:45,976] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 02:21:45,976] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 02:21:45,976] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 02:21:45,976] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 02:21:45,976] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 02:21:45,976] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 02:21:45,976] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 02:21:45,977] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 02:21:45,977] INFO : MLPF __init__ done +rank0 [2026-11-15 02:21:45,977] INFO : Instantiated model +rank0 [2026-11-15 02:21:45,978] INFO : Moving model to device rank=0 +rank0 [2026-11-15 02:21:45,989] INFO : Moved model to device rank=0 +rank0 [2026-11-15 02:21:45,990] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 02:21:46,117] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 02:21:46,117] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 02:21:46,117] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 02:21:46,215] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 02:21:46,215] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 02:21:46,216] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 02:21:46,216] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 02:21:46,216] INFO : Total parameters: 3910930 +rank0 [2026-11-15 02:21:46,218] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 02:21:46,224] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707 +rank0 [2026-11-15 02:21:46,224] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707 +rank0 [2026-11-15 02:21:46,224] INFO : Getting dataloaders +rank0 [2026-11-15 02:21:46,224] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:21:48,105] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,153] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 02:21:48,328] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 02:21:48,384] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,424] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,483] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,517] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,561] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,609] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 02:21:48,649] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 02:21:48,649] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 02:21:48,649] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 02:21:48,649] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-15 02:21:48,649] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:21:48,657] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,664] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,671] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,677] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,685] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,693] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,699] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,706] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,713] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,720] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 02:21:48,720] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 02:21:48,720] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 02:21:48,772] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:21:48,877] INFO : Got dataloaders +rank0 [2026-11-15 02:21:48,878] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-15 02:21:48,878] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-15 02:21:48,878] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 02:21:48,902] INFO : Creating train_iterator +rank0 [2026-11-15 02:21:48,902] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:21:48,924] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:21:49,037] INFO : Created train_iterator +rank0 [2026-11-15 02:22:01,450] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.643568235039711, 'Classification': 0.0195966562256217, 'Regression_pt': 0.17017275454476477, 'Regression_eta': 0.006485479580587708, 'Regression_sin_phi': 0.002063132655021036, 'Regression_cos_phi': 0.002265365930797998, 'Regression_energy': 0.17581136947497725}, weights={'Classification_binary': 1.0, 'Classification': 83.86982727050781, 'Regression_pt': 9.658233642578125, 'Regression_eta': 253.4227752685547, 'Regression_sin_phi': 796.63720703125, 'Regression_cos_phi': 725.5199584960938, 'Regression_energy': 9.348475456237793} +rank0 [2026-11-15 02:22:50,725] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 02:22:50,725] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:23:37,392] INFO : Step 1000/500000 rank0 | Train Loss: 5.3462 | Train Loss Unweighted: 1.1067 | Train Loss Components: Classification: 0.0076 | Classification_binary: 1.0055 | Regression_cos_phi: 0.0018 | Regression_energy: 0.0377 | Regression_eta: 0.0033 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.6337 | Classification_binary: 1.0055 | Regression_cos_phi: 1.3387 | Regression_energy: 0.3525 | Regression_eta: 0.8286 | Regression_pt: 0.4822 | Regression_sin_phi: 0.7050 | LR: 2.01e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4506s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0545 | max_tensorboard_logging: 0.1462 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2582.1 | uss_MB: 1347.2 | child_rss_MB: 22995.0 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:23:37,394] INFO : Running validation on rank0 +rank0 [2026-11-15 02:23:37,394] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:23:37,394] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:23:37,592] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 02:23:37,793] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:23:37,862] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:23:37,916] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:23:37,970] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:23:38,016] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:23:38,068] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:23:38,121] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:23:38,170] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:23:38,248] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:23:38,321] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:23:38,372] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:23:38,425] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:23:38,469] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:23:38,522] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:23:38,575] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:23:38,621] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:23:38,625] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:23:38,627] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:23:38,629] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:23:38,631] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:23:38,633] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:23:38,633] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:23:38,635] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:23:38,637] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:23:38,638] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:23:38,639] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:23:38,641] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:23:38,643] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:23:38,645] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:23:38,647] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:23:38,649] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:23:40,086] INFO : VALIDATION | Step=1000/500000 | Train Loss=5.3462 | Valid Loss=8.8955 | Train Loss Unweighted=1.1067 | Valid Loss Unweighted=1.3492 | Stale=0 | ETA=924.7m +rank0 [2026-11-15 02:23:40,087] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:23:40,162] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:23:40,177] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:23:40,227] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:23:40,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:23:41,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:23:41,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:23:41,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:23:41,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:23:42,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:23:42,383] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:23:42,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:23:42,850] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:23:43,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:23:43,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:23:43,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:23:43,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:23:44,054] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:23:44,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:23:44,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:23:44,556] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:23:44,562] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:23:45,087] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_1000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:23:46,006] INFO : Plotted number of elements +rank0 [2026-11-15 02:23:47,276] INFO : Plotted jets +rank0 [2026-11-15 02:23:48,615] INFO : Plotted jet ratio +rank0 [2026-11-15 02:24:02,169] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 02:24:02,169] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:25:01,908] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 02:25:01,908] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:25:35,650] INFO : Step 2000/500000 rank0 | Train Loss: 2.7241 | Train Loss Unweighted: 0.9196 | Train Loss Components: Classification: 0.0104 | Classification_binary: 0.8333 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0375 | Regression_eta: 0.0002 | Regression_pt: 0.0379 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.8682 | Classification_binary: 0.8333 | Regression_cos_phi: 0.0698 | Regression_energy: 0.3502 | Regression_eta: 0.0582 | Regression_pt: 0.3659 | Regression_sin_phi: 0.1786 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4456s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0033 | max_backward: 0.0393 | max_optimizer_step: 0.0589 | max_tensorboard_logging: 0.1277 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2714.9 | uss_MB: 1486.6 | child_rss_MB: 22765.0 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:25:35,652] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 02:25:35,716] INFO : Running validation on rank0 +rank0 [2026-11-15 02:25:35,716] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:25:35,716] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:25:35,836] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 02:25:35,940] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:25:36,041] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:25:36,111] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:25:36,177] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:25:36,240] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:25:36,305] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:25:36,370] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:25:36,479] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:25:36,589] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:25:36,698] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:25:36,765] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:25:36,830] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:25:36,894] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:25:36,959] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:25:37,023] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:25:37,135] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:25:37,142] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:25:37,144] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:25:37,146] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:25:37,148] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:25:37,150] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:25:37,151] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:25:37,153] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:25:37,154] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:25:37,155] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:25:37,157] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:25:37,159] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:25:37,160] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:25:37,161] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:25:37,162] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:25:37,165] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:25:38,544] INFO : VALIDATION | Step=2000/500000 | Train Loss=2.7241 | Valid Loss=10.2432 | Train Loss Unweighted=0.9196 | Valid Loss Unweighted=1.2907 | Stale=1 | ETA=953.0m +rank0 [2026-11-15 02:25:38,545] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:25:38,617] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:25:38,633] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:25:38,668] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:25:39,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:25:39,329] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:25:39,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:25:39,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:25:40,038] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:25:40,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:25:40,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:25:40,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:25:40,963] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:25:41,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:25:41,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:25:41,684] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:25:41,919] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:25:42,153] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:25:42,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:25:42,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:25:42,632] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:25:42,637] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:25:43,167] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_2000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:25:43,554] INFO : Plotted number of elements +rank0 [2026-11-15 02:25:44,786] INFO : Plotted jets +rank0 [2026-11-15 02:25:46,096] INFO : Plotted jet ratio +rank0 [2026-11-15 02:26:12,842] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 02:26:12,842] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:27:12,639] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 02:27:12,639] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:27:33,292] INFO : Step 3000/500000 rank0 | Train Loss: 3.3016 | Train Loss Unweighted: 0.8372 | Train Loss Components: Classification: 0.0021 | Classification_binary: 0.7160 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0623 | Regression_eta: 0.0026 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.1773 | Classification_binary: 0.7160 | Regression_cos_phi: 0.2194 | Regression_energy: 0.5820 | Regression_eta: 0.6629 | Regression_pt: 0.5155 | Regression_sin_phi: 0.4284 | LR: 2.05e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4424s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0027 | max_backward: 0.0385 | max_optimizer_step: 0.0569 | max_tensorboard_logging: 0.1270 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2773.8 | uss_MB: 1545.6 | child_rss_MB: 22765.1 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:27:33,293] INFO : Running validation on rank0 +rank0 [2026-11-15 02:27:33,294] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:27:33,294] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:27:33,414] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 02:27:33,520] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:27:33,618] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:27:33,686] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:27:33,750] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:27:33,815] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:27:33,879] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:27:33,945] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:27:34,053] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:27:34,163] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:27:34,274] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:27:34,342] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:27:34,408] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:27:34,473] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:27:34,542] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:27:34,606] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:27:34,714] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:27:34,720] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:27:34,722] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:27:34,724] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:27:34,726] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:27:34,727] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:27:34,729] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:27:34,731] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:27:34,731] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:27:34,733] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:27:34,735] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:27:34,737] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:27:34,738] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:27:34,740] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:27:34,742] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:27:34,744] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:27:36,134] INFO : VALIDATION | Step=3000/500000 | Train Loss=3.3016 | Valid Loss=14.6085 | Train Loss Unweighted=0.8372 | Valid Loss Unweighted=1.2769 | Stale=2 | ETA=958.7m +rank0 [2026-11-15 02:27:36,134] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:27:36,206] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:27:36,209] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:27:36,249] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:27:36,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:27:36,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:27:37,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:27:37,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:27:37,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:27:37,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:27:38,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:27:38,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:27:38,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:27:38,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:27:39,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:27:39,301] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:27:39,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:27:39,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:27:40,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:27:40,226] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:27:40,277] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:27:40,282] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:27:40,814] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_3000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:27:41,214] INFO : Plotted number of elements +rank0 [2026-11-15 02:27:42,434] INFO : Plotted jets +rank0 [2026-11-15 02:27:43,762] INFO : Plotted jet ratio +rank0 [2026-11-15 02:28:23,908] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 02:28:23,908] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:29:24,040] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 02:29:24,040] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:29:31,467] INFO : Step 4000/500000 rank0 | Train Loss: 4.8670 | Train Loss Unweighted: 1.8491 | Train Loss Components: Classification: 0.0035 | Classification_binary: 1.6030 | Regression_cos_phi: 0.0003 | Regression_energy: 0.1218 | Regression_eta: 0.0004 | Regression_pt: 0.1197 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.2936 | Classification_binary: 1.6030 | Regression_cos_phi: 0.2408 | Regression_energy: 1.1384 | Regression_eta: 0.1056 | Regression_pt: 1.1559 | Regression_sin_phi: 0.3297 | LR: 2.08e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4465s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0571 | max_tensorboard_logging: 0.1308 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2803.7 | uss_MB: 1575.5 | child_rss_MB: 22765.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:29:31,469] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 02:29:31,538] INFO : Running validation on rank0 +rank0 [2026-11-15 02:29:31,538] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:29:31,538] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:29:31,657] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 02:29:31,761] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:29:31,859] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:29:31,925] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:29:31,989] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:29:32,054] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:29:32,119] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:29:32,185] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:29:32,292] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:29:32,395] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:29:32,496] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:29:32,562] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:29:32,627] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:29:32,690] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:29:32,755] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:29:32,820] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:29:32,931] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:29:32,938] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:29:32,940] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:29:32,942] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:29:32,944] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:29:32,946] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:29:32,947] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:29:32,948] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:29:32,950] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:29:32,951] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:29:32,953] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:29:32,955] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:29:32,957] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:29:32,959] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:29:32,960] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:29:32,963] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:29:34,357] INFO : VALIDATION | Step=4000/500000 | Train Loss=4.8670 | Valid Loss=10.7381 | Train Loss Unweighted=1.8491 | Valid Loss Unweighted=1.2477 | Stale=3 | ETA=961.9m +rank0 [2026-11-15 02:29:34,357] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:29:34,432] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:29:34,436] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:29:34,477] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:29:34,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:29:35,129] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:29:35,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:29:35,607] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:29:35,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:29:36,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:29:36,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:29:36,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:29:36,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:29:37,028] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:29:37,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:29:37,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:29:37,739] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:29:37,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:29:38,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:29:38,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:29:38,480] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:29:38,485] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:29:39,011] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_4000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:29:39,410] INFO : Plotted number of elements +rank0 [2026-11-15 02:29:40,635] INFO : Plotted jets +rank0 [2026-11-15 02:29:41,988] INFO : Plotted jet ratio +rank0 [2026-11-15 02:30:34,990] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 02:30:34,990] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:31:29,085] INFO : Step 5000/500000 rank0 | Train Loss: 2.2699 | Train Loss Unweighted: 0.8798 | Train Loss Components: Classification: 0.0030 | Classification_binary: 0.8234 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0276 | Regression_eta: 0.0006 | Regression_pt: 0.0245 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.2484 | Classification_binary: 0.8234 | Regression_cos_phi: 0.4323 | Regression_energy: 0.2583 | Regression_eta: 0.1421 | Regression_pt: 0.2365 | Regression_sin_phi: 0.1288 | LR: 2.13e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4464s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0345 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0569 | max_tensorboard_logging: 0.1286 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2826.4 | uss_MB: 1598.3 | child_rss_MB: 22765.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:31:29,087] INFO : Running validation on rank0 +rank0 [2026-11-15 02:31:29,087] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:31:29,087] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:31:29,214] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 02:31:29,320] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:31:29,423] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:31:29,491] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:31:29,558] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:31:29,621] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:31:29,685] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:31:29,750] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:31:29,859] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:31:29,966] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:31:30,076] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:31:30,144] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:31:30,209] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:31:30,272] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:31:30,337] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:31:30,401] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:31:30,511] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:31:30,517] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:31:30,519] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:31:30,521] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:31:30,523] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:31:30,525] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:31:30,526] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:31:30,526] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:31:30,529] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:31:30,530] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:31:30,532] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:31:30,534] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:31:30,534] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:31:30,536] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:31:30,538] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:31:30,540] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:31:32,026] INFO : VALIDATION | Step=5000/500000 | Train Loss=2.2699 | Valid Loss=8.0372 | Train Loss Unweighted=0.8798 | Valid Loss Unweighted=1.2434 | Stale=0 | ETA=962.2m +rank0 [2026-11-15 02:31:32,026] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:31:32,100] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:31:32,103] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:31:32,149] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:31:32,607] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:31:32,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:31:33,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:31:33,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:31:33,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:31:33,794] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:31:34,028] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:31:34,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:31:34,492] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:31:34,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:31:34,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:31:35,198] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:31:35,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:31:35,673] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:31:35,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:31:36,133] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:31:36,188] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:31:36,193] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:31:36,716] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_5000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:31:37,115] INFO : Plotted number of elements +rank0 [2026-11-15 02:31:38,351] INFO : Plotted jets +rank0 [2026-11-15 02:31:39,690] INFO : Plotted jet ratio +rank0 [2026-11-15 02:31:46,241] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 02:31:46,241] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:32:46,501] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 02:32:46,502] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:33:27,382] INFO : Step 6000/500000 rank0 | Train Loss: 2.4642 | Train Loss Unweighted: 0.9994 | Train Loss Components: Classification: 0.0008 | Classification_binary: 0.8981 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0387 | Regression_eta: 0.0010 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.8981 | Regression_cos_phi: 0.1441 | Regression_energy: 0.3621 | Regression_eta: 0.2505 | Regression_pt: 0.5822 | Regression_sin_phi: 0.1565 | LR: 2.19e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4441s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0026 | max_backward: 0.0379 | max_optimizer_step: 0.0564 | max_tensorboard_logging: 0.1262 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2852.8 | uss_MB: 1624.6 | child_rss_MB: 22765.2 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:33:27,383] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 02:33:27,452] INFO : Running validation on rank0 +rank0 [2026-11-15 02:33:27,452] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:33:27,452] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:33:27,571] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 02:33:27,678] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:33:27,780] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:33:27,848] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:33:27,915] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:33:27,981] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:33:28,047] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:33:28,116] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:33:28,226] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:33:28,329] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:33:28,432] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:33:28,498] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:33:28,563] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:33:28,626] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:33:28,690] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:33:28,754] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:33:28,848] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:33:28,854] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:33:28,856] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:33:28,858] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:33:28,860] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:33:28,862] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:33:28,863] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:33:28,865] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:33:28,866] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:33:28,867] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:33:28,869] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:33:28,871] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:33:28,873] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:33:28,875] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:33:28,876] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:33:28,879] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:33:30,273] INFO : VALIDATION | Step=6000/500000 | Train Loss=2.4642 | Valid Loss=10.8536 | Train Loss Unweighted=0.9994 | Valid Loss Unweighted=1.2972 | Stale=1 | ETA=962.4m +rank0 [2026-11-15 02:33:30,274] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:33:30,346] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:33:30,349] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:33:30,391] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:33:30,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:33:31,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:33:31,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:33:31,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:33:31,771] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:33:32,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:33:32,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:33:32,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:33:32,721] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:33:32,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:33:33,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:33:33,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:33:33,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:33:33,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:33:34,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:33:34,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:33:34,414] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:33:34,419] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:33:34,945] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_6000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:33:35,332] INFO : Plotted number of elements +rank0 [2026-11-15 02:33:36,572] INFO : Plotted jets +rank0 [2026-11-15 02:33:37,931] INFO : Plotted jet ratio +rank0 [2026-11-15 02:33:57,588] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 02:33:57,588] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:34:57,436] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 02:34:57,436] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:35:25,025] INFO : Step 7000/500000 rank0 | Train Loss: 2.1415 | Train Loss Unweighted: 0.9811 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.8995 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0393 | Regression_eta: 0.0004 | Regression_pt: 0.0413 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0178 | Classification_binary: 0.8995 | Regression_cos_phi: 0.1180 | Regression_energy: 0.3674 | Regression_eta: 0.0957 | Regression_pt: 0.3986 | Regression_sin_phi: 0.2446 | LR: 2.26e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4570s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0354 | max_loss: 0.0027 | max_backward: 0.0398 | max_optimizer_step: 0.0602 | max_tensorboard_logging: 0.1289 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2874.5 | uss_MB: 1646.4 | child_rss_MB: 22764.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 02:35:25,027] INFO : Running validation on rank0 +rank0 [2026-11-15 02:35:25,027] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:35:25,027] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:35:25,143] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 02:35:25,246] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:35:25,346] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:35:25,414] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:35:25,478] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:35:25,542] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:35:25,607] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:35:25,672] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:35:25,779] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:35:25,883] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:35:25,987] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:35:26,054] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:35:26,119] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:35:26,184] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:35:26,250] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:35:26,316] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:35:26,412] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:35:26,418] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:35:26,420] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:35:26,422] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:35:26,424] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:35:26,426] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:35:26,427] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:35:26,427] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:35:26,430] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:35:26,431] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:35:26,433] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:35:26,435] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:35:26,436] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:35:26,437] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:35:26,439] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:35:26,441] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:35:27,825] INFO : VALIDATION | Step=7000/500000 | Train Loss=2.1415 | Valid Loss=11.9212 | Train Loss Unweighted=0.9811 | Valid Loss Unweighted=1.2719 | Stale=2 | ETA=961.3m +rank0 [2026-11-15 02:35:27,825] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:35:27,900] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:35:27,904] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:35:27,947] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:35:28,404] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:35:28,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:35:28,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:35:29,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:35:29,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:35:29,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:35:29,845] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:35:30,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:35:30,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:35:30,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:35:30,794] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:35:31,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:35:31,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:35:31,510] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:35:31,741] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:35:31,955] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:35:32,014] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:35:32,019] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:35:32,549] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_7000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:35:32,947] INFO : Plotted number of elements +rank0 [2026-11-15 02:35:34,575] INFO : Plotted jets +rank0 [2026-11-15 02:35:35,491] INFO : Plotted jet ratio +rank0 [2026-11-15 02:36:08,461] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 02:36:08,461] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:37:08,343] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 02:37:08,343] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:37:22,788] INFO : Step 8000/500000 rank0 | Train Loss: 2.1317 | Train Loss Unweighted: 0.6228 | Train Loss Components: Classification: 0.0021 | Classification_binary: 0.5271 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0499 | Regression_eta: 0.0009 | Regression_pt: 0.0425 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.1748 | Classification_binary: 0.5271 | Regression_cos_phi: 0.1956 | Regression_energy: 0.4661 | Regression_eta: 0.2387 | Regression_pt: 0.4102 | Regression_sin_phi: 0.1192 | LR: 2.34e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4667s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0373 | max_loss: 0.0030 | max_backward: 0.0428 | max_optimizer_step: 0.0615 | max_tensorboard_logging: 0.1272 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2899.4 | uss_MB: 1671.2 | child_rss_MB: 22763.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:37:22,790] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 02:37:22,858] INFO : Running validation on rank0 +rank0 [2026-11-15 02:37:22,858] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:37:22,858] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:37:22,974] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 02:37:23,079] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:37:23,177] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:37:23,245] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:37:23,311] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:37:23,375] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:37:23,441] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:37:23,507] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:37:23,618] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:37:23,729] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:37:23,837] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:37:23,905] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:37:23,970] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:37:24,036] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:37:24,103] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:37:24,169] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:37:24,263] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:37:24,269] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:37:24,271] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:37:24,273] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:37:24,275] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:37:24,277] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:37:24,277] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:37:24,279] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:37:24,281] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:37:24,282] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:37:24,284] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:37:24,285] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:37:24,288] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:37:24,289] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:37:24,291] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:37:24,293] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:37:25,679] INFO : VALIDATION | Step=8000/500000 | Train Loss=2.1317 | Valid Loss=12.9222 | Train Loss Unweighted=0.6228 | Valid Loss Unweighted=1.3010 | Stale=3 | ETA=960.2m +rank0 [2026-11-15 02:37:25,679] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:37:25,750] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:37:25,753] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:37:25,792] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:37:26,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:37:26,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:37:26,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:37:26,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:37:27,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:37:27,417] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:37:27,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:37:27,884] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:37:28,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:37:28,355] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:37:28,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:37:28,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:37:29,077] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:37:29,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:37:29,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:37:29,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:37:29,819] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:37:29,824] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:37:30,354] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_8000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:37:30,759] INFO : Plotted number of elements +rank0 [2026-11-15 02:37:31,998] INFO : Plotted jets +rank0 [2026-11-15 02:37:33,346] INFO : Plotted jet ratio +rank0 [2026-11-15 02:38:19,841] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 02:38:19,841] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:39:19,807] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 02:39:19,807] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:39:21,019] INFO : Step 9000/500000 rank0 | Train Loss: 0.9153 | Train Loss Unweighted: 0.4778 | Train Loss Components: Classification: 0.0007 | Classification_binary: 0.4546 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0109 | Regression_eta: 0.0002 | Regression_pt: 0.0113 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.4546 | Regression_cos_phi: 0.0681 | Regression_energy: 0.1019 | Regression_eta: 0.0457 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0800 | LR: 2.43e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4549s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0575 | max_tensorboard_logging: 0.1371 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2853.0 | uss_MB: 1624.9 | child_rss_MB: 22759.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:39:21,020] INFO : Running validation on rank0 +rank0 [2026-11-15 02:39:21,020] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:39:21,020] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:39:21,156] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 02:39:21,262] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:39:21,361] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:39:21,430] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:39:21,495] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:39:21,558] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:39:21,622] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:39:21,687] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:39:21,795] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:39:21,903] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:39:22,009] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:39:22,076] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:39:22,140] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:39:22,204] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:39:22,269] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:39:22,335] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:39:22,431] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:39:22,436] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:39:22,438] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:39:22,440] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:39:22,442] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:39:22,444] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:39:22,445] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:39:22,447] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:39:22,449] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:39:22,449] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:39:22,451] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:39:22,453] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:39:22,454] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:39:22,454] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:39:22,457] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:39:22,460] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:39:23,843] INFO : VALIDATION | Step=9000/500000 | Train Loss=0.9153 | Valid Loss=8.1332 | Train Loss Unweighted=0.4778 | Valid Loss Unweighted=1.2939 | Stale=4 | ETA=959.2m +rank0 [2026-11-15 02:39:23,844] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:39:23,915] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:39:23,929] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:39:23,972] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:39:24,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:39:24,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:39:24,906] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:39:25,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:39:25,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:39:25,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:39:25,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:39:26,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:39:26,329] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:39:26,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:39:26,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:39:27,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:39:27,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:39:27,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:39:27,759] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:39:27,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:39:28,024] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:39:28,041] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:39:28,566] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_9000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:39:28,982] INFO : Plotted number of elements +rank0 [2026-11-15 02:39:30,221] INFO : Plotted jets +rank0 [2026-11-15 02:39:31,562] INFO : Plotted jet ratio +rank0 [2026-11-15 02:40:30,948] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 02:40:30,948] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:41:18,920] INFO : Step 10000/500000 rank0 | Train Loss: 1.1075 | Train Loss Unweighted: 0.4842 | Train Loss Components: Classification: 0.0008 | Classification_binary: 0.4421 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0206 | Regression_eta: 0.0001 | Regression_pt: 0.0204 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.4421 | Regression_cos_phi: 0.1057 | Regression_energy: 0.1922 | Regression_eta: 0.0307 | Regression_pt: 0.1970 | Regression_sin_phi: 0.0768 | LR: 2.52e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4574s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0387 | max_optimizer_step: 0.0591 | max_tensorboard_logging: 0.1275 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2950.1 | uss_MB: 1722.0 | child_rss_MB: 22762.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:41:18,922] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 02:41:18,989] INFO : Running validation on rank0 +rank0 [2026-11-15 02:41:18,989] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:41:18,989] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:41:19,118] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 02:41:19,223] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:41:19,327] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:41:19,394] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:41:19,459] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:41:19,523] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:41:19,591] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:41:19,657] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:41:19,765] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:41:19,868] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:41:19,972] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:41:20,038] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:41:20,102] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:41:20,166] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:41:20,231] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:41:20,296] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:41:20,393] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:41:20,399] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:41:20,401] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:41:20,403] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:41:20,405] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:41:20,407] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:41:20,408] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:41:20,409] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:41:20,411] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:41:20,412] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:41:20,414] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:41:20,416] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:41:20,418] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:41:20,419] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:41:20,421] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:41:20,423] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:41:21,836] INFO : VALIDATION | Step=10000/500000 | Train Loss=1.1075 | Valid Loss=8.1656 | Train Loss Unweighted=0.4842 | Valid Loss Unweighted=1.3168 | Stale=5 | ETA=957.9m +rank0 [2026-11-15 02:41:21,836] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:41:21,910] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:41:21,914] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:41:21,955] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:41:22,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:41:22,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:41:22,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:41:23,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:41:23,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:41:23,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:41:23,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:41:24,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:41:24,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:41:24,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:41:24,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:41:25,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:41:25,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:41:25,476] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:41:25,712] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:41:25,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:41:25,988] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:41:25,994] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:41:26,517] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:41:26,914] INFO : Plotted number of elements +rank0 [2026-11-15 02:41:28,137] INFO : Plotted jets +rank0 [2026-11-15 02:41:29,466] INFO : Plotted jet ratio +rank0 [2026-11-15 02:41:42,151] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 02:41:42,151] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:42:42,070] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 02:42:42,070] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:43:16,809] INFO : Step 11000/500000 rank0 | Train Loss: 1.0296 | Train Loss Unweighted: 0.3118 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2535 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0288 | Regression_eta: 0.0001 | Regression_pt: 0.0286 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.2535 | Regression_cos_phi: 0.0514 | Regression_energy: 0.2697 | Regression_eta: 0.0301 | Regression_pt: 0.2766 | Regression_sin_phi: 0.1106 | LR: 2.63e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4653s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0026 | max_backward: 0.0385 | max_optimizer_step: 0.0591 | max_tensorboard_logging: 0.1347 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2977.2 | uss_MB: 1749.1 | child_rss_MB: 22767.2 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 12.0 +rank0 [2026-11-15 02:43:16,811] INFO : Running validation on rank0 +rank0 [2026-11-15 02:43:16,811] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:43:16,811] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:43:16,943] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 02:43:17,048] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:43:17,151] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:43:17,221] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:43:17,287] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:43:17,353] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:43:17,420] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:43:17,487] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:43:17,596] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:43:17,699] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:43:17,801] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:43:17,867] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:43:17,932] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:43:17,994] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:43:18,058] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:43:18,122] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:43:18,216] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:43:18,221] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:43:18,223] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:43:18,224] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:43:18,228] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:43:18,229] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:43:18,230] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:43:18,232] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:43:18,234] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:43:18,234] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:43:18,236] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:43:18,238] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:43:18,239] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:43:18,239] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:43:18,243] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:43:18,245] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:43:19,644] INFO : VALIDATION | Step=11000/500000 | Train Loss=1.0296 | Valid Loss=14.4730 | Train Loss Unweighted=0.3118 | Valid Loss Unweighted=1.3514 | Stale=6 | ETA=956.3m +rank0 [2026-11-15 02:43:19,645] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:43:19,721] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:43:19,724] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:43:19,763] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:43:20,189] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:43:20,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:43:20,670] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:43:20,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:43:21,142] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:43:21,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:43:21,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:43:21,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:43:22,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:43:22,337] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:43:22,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:43:22,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:43:23,043] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:43:23,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:43:23,526] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:43:23,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:43:23,802] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:43:23,808] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:43:24,335] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_11000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:43:24,723] INFO : Plotted number of elements +rank0 [2026-11-15 02:43:25,970] INFO : Plotted jets +rank0 [2026-11-15 02:43:27,305] INFO : Plotted jet ratio +rank0 [2026-11-15 02:43:53,316] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 02:43:53,316] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:44:53,514] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 02:44:53,514] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:45:15,085] INFO : Step 12000/500000 rank0 | Train Loss: 1.4650 | Train Loss Unweighted: 0.3714 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.3256 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0219 | Regression_eta: 0.0002 | Regression_pt: 0.0226 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0219 | Classification_binary: 0.3256 | Regression_cos_phi: 0.4388 | Regression_energy: 0.2044 | Regression_eta: 0.0486 | Regression_pt: 0.2181 | Regression_sin_phi: 0.2076 | LR: 2.75e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4528s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0579 | max_tensorboard_logging: 0.1269 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2984.0 | uss_MB: 1755.8 | child_rss_MB: 22766.1 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:45:15,086] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 02:45:15,152] INFO : Running validation on rank0 +rank0 [2026-11-15 02:45:15,152] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:45:15,152] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:45:15,273] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 02:45:15,379] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:45:15,480] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:45:15,549] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:45:15,614] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:45:15,679] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:45:15,747] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:45:15,814] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:45:15,923] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:45:16,026] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:45:16,128] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:45:16,195] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:45:16,258] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:45:16,322] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:45:16,387] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:45:16,452] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:45:16,546] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:45:16,552] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:45:16,554] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:45:16,556] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:45:16,558] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:45:16,560] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:45:16,561] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:45:16,561] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:45:16,565] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:45:16,565] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:45:16,567] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:45:16,569] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:45:16,570] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:45:16,570] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:45:16,574] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:45:16,576] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:45:17,970] INFO : VALIDATION | Step=12000/500000 | Train Loss=1.4650 | Valid Loss=11.5311 | Train Loss Unweighted=0.3714 | Valid Loss Unweighted=1.3716 | Stale=7 | ETA=955.0m +rank0 [2026-11-15 02:45:17,970] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:45:18,045] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:45:18,061] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:45:18,105] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:45:18,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:45:18,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:45:19,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:45:19,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:45:19,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:45:19,766] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:45:20,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:45:20,256] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:45:20,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:45:20,739] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:45:20,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:45:21,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:45:21,448] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:45:21,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:45:21,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:45:22,153] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:45:22,196] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:45:22,203] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:45:22,737] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_12000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:45:23,130] INFO : Plotted number of elements +rank0 [2026-11-15 02:45:24,389] INFO : Plotted jets +rank0 [2026-11-15 02:45:25,725] INFO : Plotted jet ratio +rank0 [2026-11-15 02:46:05,159] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 02:46:05,159] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:47:05,252] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 02:47:05,252] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:47:13,574] INFO : Step 13000/500000 rank0 | Train Loss: 1.7463 | Train Loss Unweighted: 0.8769 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.8029 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0367 | Regression_eta: 0.0002 | Regression_pt: 0.0366 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0264 | Classification_binary: 0.8029 | Regression_cos_phi: 0.0661 | Regression_energy: 0.3427 | Regression_eta: 0.0526 | Regression_pt: 0.3537 | Regression_sin_phi: 0.1019 | LR: 2.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4741s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0382 | max_optimizer_step: 0.0590 | max_tensorboard_logging: 0.1260 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3019.2 | uss_MB: 1791.1 | child_rss_MB: 22765.9 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:47:13,576] INFO : Running validation on rank0 +rank0 [2026-11-15 02:47:13,576] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:47:13,576] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:47:13,705] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 02:47:13,813] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:47:13,912] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:47:13,981] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:47:14,046] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:47:14,109] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:47:14,174] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:47:14,239] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:47:14,350] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:47:14,459] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:47:14,569] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:47:14,638] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:47:14,704] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:47:14,769] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:47:14,835] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:47:14,900] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:47:14,995] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:47:15,001] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:47:15,001] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:47:15,005] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:47:15,005] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:47:15,007] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:47:15,009] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:47:15,011] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:47:15,013] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:47:15,014] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:47:15,016] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:47:15,018] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:47:15,020] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:47:15,022] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:47:15,024] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:47:15,026] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:47:16,421] INFO : VALIDATION | Step=13000/500000 | Train Loss=1.7463 | Valid Loss=14.5748 | Train Loss Unweighted=0.8769 | Valid Loss Unweighted=1.3713 | Stale=8 | ETA=953.7m +rank0 [2026-11-15 02:47:16,422] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:47:16,492] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:47:16,497] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:47:16,540] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:47:16,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:47:17,212] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:47:17,447] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:47:17,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:47:17,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:47:18,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:47:18,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:47:18,635] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:47:18,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:47:19,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:47:19,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:47:19,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:47:19,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:47:20,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:47:20,290] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:47:20,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:47:20,562] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:47:20,568] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:47:21,090] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_13000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:47:21,486] INFO : Plotted number of elements +rank0 [2026-11-15 02:47:22,710] INFO : Plotted jets +rank0 [2026-11-15 02:47:24,049] INFO : Plotted jet ratio +rank0 [2026-11-15 02:48:16,272] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 02:48:16,272] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:49:11,172] INFO : Step 14000/500000 rank0 | Train Loss: 3.6761 | Train Loss Unweighted: 0.6192 | Train Loss Components: Classification: 0.0010 | Classification_binary: 0.3692 | Regression_cos_phi: 0.0001 | Regression_energy: 0.1224 | Regression_eta: 0.0028 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0801 | Classification_binary: 0.3692 | Regression_cos_phi: 0.1063 | Regression_energy: 1.1441 | Regression_eta: 0.7178 | Regression_pt: 1.1942 | Regression_sin_phi: 0.0644 | LR: 3.02e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4606s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0343 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0590 | max_tensorboard_logging: 0.1320 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3051.1 | uss_MB: 1823.0 | child_rss_MB: 22766.5 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 02:49:11,178] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 02:49:11,244] INFO : Running validation on rank0 +rank0 [2026-11-15 02:49:11,244] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:49:11,244] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:49:11,365] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 02:49:11,474] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:49:11,576] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:49:11,646] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:49:11,713] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:49:11,778] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:49:11,845] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:49:11,912] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:49:12,023] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:49:12,128] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:49:12,230] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:49:12,297] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:49:12,362] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:49:12,426] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:49:12,490] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:49:12,555] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:49:12,649] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:49:12,655] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:49:12,657] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:49:12,660] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:49:12,661] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:49:12,663] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:49:12,664] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:49:12,666] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:49:12,668] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:49:12,668] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:49:12,670] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:49:12,672] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:49:12,673] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:49:12,673] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:49:12,675] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:49:12,679] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:49:14,069] INFO : VALIDATION | Step=14000/500000 | Train Loss=3.6761 | Valid Loss=14.8881 | Train Loss Unweighted=0.6192 | Valid Loss Unweighted=1.4286 | Stale=9 | ETA=951.8m +rank0 [2026-11-15 02:49:14,069] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:49:14,145] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:49:14,148] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:49:14,190] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:49:14,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:49:14,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:49:15,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:49:15,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:49:15,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:49:15,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:49:16,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:49:16,307] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:49:16,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:49:16,789] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:49:17,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:49:17,260] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:49:17,496] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:49:17,733] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:49:17,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:49:18,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:49:18,247] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:49:18,254] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:49:18,778] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_14000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:49:19,165] INFO : Plotted number of elements +rank0 [2026-11-15 02:49:20,425] INFO : Plotted jets +rank0 [2026-11-15 02:49:21,836] INFO : Plotted jet ratio +rank0 [2026-11-15 02:49:27,466] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 02:49:27,466] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:50:27,331] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 02:50:27,332] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:51:08,968] INFO : Step 15000/500000 rank0 | Train Loss: 0.6973 | Train Loss Unweighted: 0.1694 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1410 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0137 | Regression_eta: 0.0003 | Regression_pt: 0.0141 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0015 | Classification_binary: 0.1410 | Regression_cos_phi: 0.0974 | Regression_energy: 0.1279 | Regression_eta: 0.0735 | Regression_pt: 0.1364 | Regression_sin_phi: 0.1196 | LR: 3.17e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4613s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0369 | max_loss: 0.0031 | max_backward: 0.0390 | max_optimizer_step: 0.0604 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3072.5 | uss_MB: 1844.4 | child_rss_MB: 22765.2 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:51:08,970] INFO : Running validation on rank0 +rank0 [2026-11-15 02:51:08,970] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:51:08,970] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:51:09,087] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 02:51:09,191] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:51:09,290] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:51:09,359] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:51:09,424] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:51:09,488] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:51:09,554] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:51:09,619] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:51:09,731] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:51:09,842] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:51:09,952] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:51:10,019] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:51:10,085] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:51:10,151] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:51:10,217] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:51:10,281] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:51:10,376] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:51:10,382] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:51:10,384] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:51:10,386] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:51:10,388] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:51:10,390] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:51:10,391] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:51:10,393] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:51:10,395] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:51:10,395] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:51:10,397] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:51:10,399] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:51:10,401] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:51:10,403] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:51:10,405] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:51:10,407] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:51:11,788] INFO : VALIDATION | Step=15000/500000 | Train Loss=0.6973 | Valid Loss=10.6270 | Train Loss Unweighted=0.1694 | Valid Loss Unweighted=1.5030 | Stale=10 | ETA=950.0m +rank0 [2026-11-15 02:51:11,788] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:51:11,863] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:51:11,866] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:51:11,911] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:51:12,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:51:12,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:51:12,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:51:13,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:51:13,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:51:13,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:51:13,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:51:14,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:51:14,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:51:14,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:51:14,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:51:15,038] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:51:15,275] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:51:15,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:51:15,741] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:51:15,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:51:16,008] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:51:16,013] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:51:16,550] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_15000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:51:16,939] INFO : Plotted number of elements +rank0 [2026-11-15 02:51:18,174] INFO : Plotted jets +rank0 [2026-11-15 02:51:19,515] INFO : Plotted jet ratio +rank0 [2026-11-15 02:51:38,472] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 02:51:38,473] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:52:38,661] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 02:52:38,662] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:53:07,427] INFO : Step 16000/500000 rank0 | Train Loss: 0.5156 | Train Loss Unweighted: 0.1593 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1359 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0116 | Regression_eta: 0.0001 | Regression_pt: 0.0116 | Regression_sin_phi: 0.0000 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0003 | Classification_binary: 0.1359 | Regression_cos_phi: 0.1014 | Regression_energy: 0.1086 | Regression_eta: 0.0376 | Regression_pt: 0.1116 | Regression_sin_phi: 0.0203 | LR: 3.33e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4638s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0366 | max_loss: 0.0026 | max_backward: 0.0380 | max_optimizer_step: 0.0591 | max_tensorboard_logging: 0.1313 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3092.2 | uss_MB: 1864.1 | child_rss_MB: 22764.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:53:07,429] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 02:53:07,496] INFO : Running validation on rank0 +rank0 [2026-11-15 02:53:07,496] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:53:07,496] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:53:07,620] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 02:53:07,727] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:53:07,829] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:53:07,897] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:53:07,961] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:53:08,026] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:53:08,092] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:53:08,157] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:53:08,269] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:53:08,376] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:53:08,485] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:53:08,550] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:53:08,614] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:53:08,678] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:53:08,743] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:53:08,808] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:53:08,903] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:53:08,908] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:53:08,911] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:53:08,913] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:53:08,915] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:53:08,916] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:53:08,917] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:53:08,919] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:53:08,921] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:53:08,922] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:53:08,923] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:53:08,924] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:53:08,926] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:53:08,928] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:53:08,930] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:53:08,931] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:53:10,321] INFO : VALIDATION | Step=16000/500000 | Train Loss=0.5156 | Valid Loss=9.3582 | Train Loss Unweighted=0.1593 | Valid Loss Unweighted=1.5123 | Stale=11 | ETA=948.5m +rank0 [2026-11-15 02:53:10,322] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:53:10,395] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:53:10,398] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:53:10,442] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:53:10,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:53:11,115] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:53:11,352] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:53:11,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:53:11,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:53:12,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:53:12,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:53:12,555] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:53:12,789] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:53:13,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:53:13,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:53:13,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:53:13,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:53:14,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:53:14,244] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:53:14,457] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:53:14,517] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:53:14,522] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:53:15,050] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_16000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:53:15,452] INFO : Plotted number of elements +rank0 [2026-11-15 02:53:16,706] INFO : Plotted jets +rank0 [2026-11-15 02:53:18,100] INFO : Plotted jet ratio +rank0 [2026-11-15 02:53:50,506] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 02:53:50,506] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:54:51,205] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 02:54:51,205] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:55:06,572] INFO : Step 17000/500000 rank0 | Train Loss: 1.1260 | Train Loss Unweighted: 0.3999 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.3402 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0298 | Regression_eta: 0.0002 | Regression_pt: 0.0295 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0029 | Classification_binary: 0.3402 | Regression_cos_phi: 0.0912 | Regression_energy: 0.2782 | Regression_eta: 0.0471 | Regression_pt: 0.2852 | Regression_sin_phi: 0.0813 | LR: 3.51e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4650s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0030 | max_backward: 0.0397 | max_optimizer_step: 0.0608 | max_tensorboard_logging: 0.1302 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3119.4 | uss_MB: 1891.3 | child_rss_MB: 22758.5 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 02:55:06,573] INFO : Running validation on rank0 +rank0 [2026-11-15 02:55:06,573] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:55:06,573] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:55:06,692] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 02:55:06,802] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:55:06,906] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:55:06,976] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:55:07,045] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:55:07,111] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:55:07,176] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:55:07,241] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:55:07,353] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:55:07,465] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:55:07,578] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:55:07,648] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:55:07,716] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:55:07,783] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:55:07,850] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:55:07,918] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:55:08,013] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:55:08,019] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:55:08,021] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:55:08,023] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:55:08,025] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:55:08,025] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:55:08,027] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:55:08,029] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:55:08,032] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:55:08,032] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:55:08,033] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:55:08,036] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:55:08,037] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:55:08,039] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:55:08,041] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:55:08,043] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:55:09,430] INFO : VALIDATION | Step=17000/500000 | Train Loss=1.1260 | Valid Loss=15.6847 | Train Loss Unweighted=0.3999 | Valid Loss Unweighted=1.5840 | Stale=12 | ETA=947.3m +rank0 [2026-11-15 02:55:09,430] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:55:09,505] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:55:09,508] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:55:09,551] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:55:09,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:55:10,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:55:10,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:55:10,714] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:55:10,953] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:55:11,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:55:11,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:55:11,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:55:11,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:55:12,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:55:12,377] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:55:12,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:55:12,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:55:13,111] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:55:13,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:55:13,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:55:13,616] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:55:13,622] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:55:14,156] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_17000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:55:14,559] INFO : Plotted number of elements +rank0 [2026-11-15 02:55:15,792] INFO : Plotted jets +rank0 [2026-11-15 02:55:17,160] INFO : Plotted jet ratio +rank0 [2026-11-15 02:56:03,082] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 02:56:03,082] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:57:03,566] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 02:57:03,566] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:57:05,668] INFO : Step 18000/500000 rank0 | Train Loss: 0.6758 | Train Loss Unweighted: 0.2777 | Train Loss Components: Classification: 0.0010 | Classification_binary: 0.2563 | Regression_cos_phi: 0.0000 | Regression_energy: 0.0101 | Regression_eta: 0.0001 | Regression_pt: 0.0101 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0807 | Classification_binary: 0.2563 | Regression_cos_phi: 0.0295 | Regression_energy: 0.0947 | Regression_eta: 0.0267 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0907 | LR: 3.69e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4624s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0394 | max_optimizer_step: 0.0612 | max_tensorboard_logging: 0.1280 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3142.9 | uss_MB: 1914.8 | child_rss_MB: 22766.1 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 02:57:05,670] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 02:57:05,735] INFO : Running validation on rank0 +rank0 [2026-11-15 02:57:05,735] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:57:05,736] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:57:05,854] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 02:57:05,960] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:57:06,061] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:57:06,128] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:57:06,194] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:57:06,261] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:57:06,328] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:57:06,396] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:57:06,509] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:57:06,620] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:57:06,730] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:57:06,797] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:57:06,862] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:57:06,926] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:57:06,991] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:57:07,057] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:57:07,152] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:57:07,158] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:57:07,160] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:57:07,161] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:57:07,164] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:57:07,165] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:57:07,167] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:57:07,168] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:57:07,170] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:57:07,171] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:57:07,173] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:57:07,175] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:57:07,175] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:57:07,177] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:57:07,179] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:57:07,181] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:57:08,581] INFO : VALIDATION | Step=18000/500000 | Train Loss=0.6758 | Valid Loss=15.0661 | Train Loss Unweighted=0.2777 | Valid Loss Unweighted=1.6344 | Stale=13 | ETA=946.0m +rank0 [2026-11-15 02:57:08,581] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:57:08,655] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:57:08,659] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:57:08,703] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:57:09,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:57:09,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:57:09,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:57:09,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:57:10,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:57:10,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:57:10,594] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:57:10,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:57:11,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:57:11,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:57:11,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:57:11,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:57:12,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:57:12,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:57:12,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:57:12,751] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:57:12,808] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 02:57:12,813] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:57:13,343] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_18000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:57:13,731] INFO : Plotted number of elements +rank0 [2026-11-15 02:57:14,973] INFO : Plotted jets +rank0 [2026-11-15 02:57:16,322] INFO : Plotted jet ratio +rank0 [2026-11-15 02:58:15,637] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 02:58:15,637] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:59:04,611] INFO : Step 19000/500000 rank0 | Train Loss: 1.4326 | Train Loss Unweighted: 0.7073 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.6686 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0179 | Regression_eta: 0.0008 | Regression_pt: 0.0193 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.6686 | Regression_cos_phi: 0.0615 | Regression_energy: 0.1675 | Regression_eta: 0.2139 | Regression_pt: 0.1864 | Regression_sin_phi: 0.0994 | LR: 3.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4547s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0383 | max_loss: 0.0026 | max_backward: 0.0389 | max_optimizer_step: 0.0595 | max_tensorboard_logging: 0.1271 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3169.0 | uss_MB: 1940.9 | child_rss_MB: 22764.0 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 02:59:04,612] INFO : Running validation on rank0 +rank0 [2026-11-15 02:59:04,612] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:59:04,612] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:59:04,731] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 02:59:04,834] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:59:04,933] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:59:05,002] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:59:05,067] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:59:05,131] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:59:05,196] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:59:05,264] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:59:05,376] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:59:05,479] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:59:05,581] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:59:05,647] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:59:05,713] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:59:05,778] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:59:05,845] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:59:05,908] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:59:06,003] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:59:06,007] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:59:06,009] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:59:06,013] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:59:06,015] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:59:06,015] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:59:06,017] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:59:06,019] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:59:06,020] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:59:06,021] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:59:06,023] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:59:06,025] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:59:06,026] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:59:06,027] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:59:06,028] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:59:06,032] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:59:07,420] INFO : VALIDATION | Step=19000/500000 | Train Loss=1.4326 | Valid Loss=13.0207 | Train Loss Unweighted=0.7073 | Valid Loss Unweighted=1.6752 | Stale=14 | ETA=944.5m +rank0 [2026-11-15 02:59:07,420] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 02:59:07,492] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 02:59:07,497] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 02:59:07,541] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:59:07,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 02:59:08,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 02:59:08,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 02:59:08,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 02:59:08,928] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 02:59:09,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 02:59:09,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 02:59:09,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 02:59:09,859] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 02:59:10,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 02:59:10,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 02:59:10,574] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 02:59:10,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 02:59:11,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 02:59:11,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 02:59:11,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 02:59:11,552] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 02:59:11,558] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:59:12,094] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_19000/cld_edm_z_tautau_pf +rank0 [2026-11-15 02:59:12,493] INFO : Plotted number of elements +rank0 [2026-11-15 02:59:13,736] INFO : Plotted jets +rank0 [2026-11-15 02:59:15,106] INFO : Plotted jet ratio +rank0 [2026-11-15 02:59:27,006] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 02:59:27,007] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:00:27,217] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 03:00:27,217] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:01:02,944] INFO : Step 20000/500000 rank0 | Train Loss: 0.6748 | Train Loss Unweighted: 0.1262 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0824 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0215 | Regression_eta: 0.0001 | Regression_pt: 0.0220 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0012 | Classification_binary: 0.0824 | Regression_cos_phi: 0.0560 | Regression_energy: 0.2009 | Regression_eta: 0.0214 | Regression_pt: 0.2129 | Regression_sin_phi: 0.1000 | LR: 4.07e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4618s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0026 | max_backward: 0.0387 | max_optimizer_step: 0.0587 | max_tensorboard_logging: 0.1305 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3123.2 | uss_MB: 1895.1 | child_rss_MB: 22766.0 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:01:02,946] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 03:01:03,018] INFO : Running validation on rank0 +rank0 [2026-11-15 03:01:03,018] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:01:03,018] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:01:03,144] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 03:01:03,255] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:01:03,357] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:01:03,425] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:01:03,490] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:01:03,553] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:01:03,618] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:01:03,683] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:01:03,793] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:01:03,906] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:01:04,017] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:01:04,089] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:01:04,156] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:01:04,222] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:01:04,287] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:01:04,353] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:01:04,451] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:01:04,457] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:01:04,459] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:01:04,461] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:01:04,463] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:01:04,465] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:01:04,466] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:01:04,468] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:01:04,470] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:01:04,470] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:01:04,472] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:01:04,474] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:01:04,475] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:01:04,476] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:01:04,479] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:01:04,481] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:01:05,871] INFO : VALIDATION | Step=20000/500000 | Train Loss=0.6748 | Valid Loss=13.2424 | Train Loss Unweighted=0.1262 | Valid Loss Unweighted=1.7804 | Stale=15 | ETA=942.8m +rank0 [2026-11-15 03:01:05,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:01:05,943] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:01:05,946] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:01:05,990] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:01:06,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:01:06,661] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:01:06,899] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:01:07,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:01:07,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:01:07,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:01:07,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:01:08,105] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:01:08,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:01:08,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:01:08,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:01:09,053] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:01:09,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:01:09,534] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:01:09,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:01:09,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:01:10,036] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:01:10,042] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:01:10,568] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:01:10,967] INFO : Plotted number of elements +rank0 [2026-11-15 03:01:12,216] INFO : Plotted jets +rank0 [2026-11-15 03:01:13,571] INFO : Plotted jet ratio +rank0 [2026-11-15 03:01:38,839] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 03:01:38,839] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:02:39,020] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 03:02:39,020] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:03:01,465] INFO : Step 21000/500000 rank0 | Train Loss: 1.0997 | Train Loss Unweighted: 0.8422 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.8260 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0081 | Regression_eta: 0.0001 | Regression_pt: 0.0078 | Regression_sin_phi: 0.0000 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0013 | Classification_binary: 0.8260 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0761 | Regression_eta: 0.0296 | Regression_pt: 0.0752 | Regression_sin_phi: 0.0301 | LR: 4.28e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4643s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0385 | max_optimizer_step: 0.0602 | max_tensorboard_logging: 0.1324 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3215.8 | uss_MB: 1987.7 | child_rss_MB: 22766.2 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:03:01,467] INFO : Running validation on rank0 +rank0 [2026-11-15 03:03:01,467] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:03:01,467] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:03:01,592] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 03:03:01,698] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:03:01,798] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:03:01,866] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:03:01,931] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:03:01,994] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:03:02,059] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:03:02,124] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:03:02,234] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:03:02,335] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:03:02,436] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:03:02,502] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:03:02,566] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:03:02,631] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:03:02,696] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:03:02,761] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:03:02,861] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:03:02,867] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:03:02,869] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:03:02,871] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:03:02,873] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:03:02,875] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:03:02,876] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:03:02,876] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:03:02,879] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:03:02,880] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:03:02,882] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:03:02,884] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:03:02,885] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:03:02,886] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:03:02,888] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:03:02,890] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:03:04,283] INFO : VALIDATION | Step=21000/500000 | Train Loss=1.0997 | Valid Loss=16.9017 | Train Loss Unweighted=0.8422 | Valid Loss Unweighted=1.9002 | Stale=16 | ETA=941.0m +rank0 [2026-11-15 03:03:04,284] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:03:04,357] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:03:04,360] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:03:04,404] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:03:04,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:03:05,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:03:05,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:03:05,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:03:05,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:03:06,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:03:06,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:03:06,523] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:03:06,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:03:06,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:03:07,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:03:07,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:03:07,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:03:07,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:03:08,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:03:08,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:03:08,496] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:03:08,502] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:03:09,027] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_21000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:03:09,456] INFO : Plotted number of elements +rank0 [2026-11-15 03:03:10,679] INFO : Plotted jets +rank0 [2026-11-15 03:03:12,041] INFO : Plotted jet ratio +rank0 [2026-11-15 03:03:50,583] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 03:03:50,584] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:04:50,832] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 03:04:50,832] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:05:00,006] INFO : Step 22000/500000 rank0 | Train Loss: 0.4749 | Train Loss Unweighted: 0.0815 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0578 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0121 | Regression_eta: 0.0003 | Regression_pt: 0.0112 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0006 | Classification_binary: 0.0578 | Regression_cos_phi: 0.0744 | Regression_energy: 0.1128 | Regression_eta: 0.0669 | Regression_pt: 0.1082 | Regression_sin_phi: 0.0543 | LR: 4.50e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4633s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0385 | max_optimizer_step: 0.0594 | max_tensorboard_logging: 0.1314 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3249.7 | uss_MB: 2021.6 | child_rss_MB: 22766.2 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 12.0 +rank0 [2026-11-15 03:05:00,007] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 03:05:00,075] INFO : Running validation on rank0 +rank0 [2026-11-15 03:05:00,075] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:05:00,075] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:05:00,195] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 03:05:00,300] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:05:00,401] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:05:00,468] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:05:00,532] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:05:00,596] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:05:00,661] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:05:00,726] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:05:00,838] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:05:00,946] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:05:01,054] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:05:01,122] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:05:01,187] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:05:01,250] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:05:01,315] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:05:01,380] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:05:01,478] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:05:01,484] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:05:01,487] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:05:01,489] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:05:01,491] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:05:01,493] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:05:01,493] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:05:01,494] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:05:01,497] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:05:01,498] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:05:01,500] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:05:01,502] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:05:01,502] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:05:01,504] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:05:01,506] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:05:01,508] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:05:02,885] INFO : VALIDATION | Step=22000/500000 | Train Loss=0.4749 | Valid Loss=15.5750 | Train Loss Unweighted=0.0815 | Valid Loss Unweighted=1.9360 | Stale=17 | ETA=939.3m +rank0 [2026-11-15 03:05:02,885] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:05:02,966] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:05:02,969] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:05:03,012] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:05:03,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:05:03,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:05:03,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:05:04,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:05:04,438] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:05:04,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:05:04,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:05:05,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:05:05,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:05:05,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:05:05,881] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:05:06,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:05:06,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:05:06,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:05:06,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:05:07,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:05:07,110] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:05:07,115] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:05:07,648] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_22000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:05:08,039] INFO : Plotted number of elements +rank0 [2026-11-15 03:05:09,297] INFO : Plotted jets +rank0 [2026-11-15 03:05:10,701] INFO : Plotted jet ratio +rank0 [2026-11-15 03:06:02,714] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 03:06:02,714] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:06:58,885] INFO : Step 23000/500000 rank0 | Train Loss: 1.1171 | Train Loss Unweighted: 0.3516 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.2971 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0262 | Regression_eta: 0.0002 | Regression_pt: 0.0277 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0010 | Classification_binary: 0.2971 | Regression_cos_phi: 0.1174 | Regression_energy: 0.2453 | Regression_eta: 0.0555 | Regression_pt: 0.2674 | Regression_sin_phi: 0.1334 | LR: 4.73e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4579s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0390 | max_optimizer_step: 0.0590 | max_tensorboard_logging: 0.1263 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3261.5 | uss_MB: 2033.4 | child_rss_MB: 22765.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:06:58,887] INFO : Running validation on rank0 +rank0 [2026-11-15 03:06:58,887] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:06:58,887] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:06:59,011] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 03:06:59,123] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:06:59,228] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:06:59,298] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:06:59,365] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:06:59,431] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:06:59,499] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:06:59,565] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:06:59,677] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:06:59,789] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:06:59,901] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:06:59,968] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:07:00,035] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:07:00,103] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:07:00,168] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:07:00,233] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:07:00,331] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:07:00,336] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:07:00,338] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:07:00,341] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:07:00,343] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:07:00,343] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:07:00,345] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:07:00,347] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:07:00,349] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:07:00,349] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:07:00,351] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:07:00,353] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:07:00,355] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:07:00,357] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:07:00,359] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:07:00,360] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:07:01,748] INFO : VALIDATION | Step=23000/500000 | Train Loss=1.1171 | Valid Loss=18.5581 | Train Loss Unweighted=0.3516 | Valid Loss Unweighted=2.0273 | Stale=18 | ETA=937.7m +rank0 [2026-11-15 03:07:01,748] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:07:01,820] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:07:01,823] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:07:01,867] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:07:02,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:07:02,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:07:02,810] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:07:03,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:07:03,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:07:03,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:07:03,759] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:07:04,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:07:04,245] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:07:04,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:07:04,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:07:04,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:07:05,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:07:05,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:07:05,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:07:05,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:07:05,943] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:07:05,948] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:07:06,477] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_23000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:07:06,865] INFO : Plotted number of elements +rank0 [2026-11-15 03:07:08,110] INFO : Plotted jets +rank0 [2026-11-15 03:07:09,458] INFO : Plotted jet ratio +rank0 [2026-11-15 03:07:14,314] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 03:07:14,314] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:08:14,715] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 03:08:14,715] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:08:57,660] INFO : Step 24000/500000 rank0 | Train Loss: 1.0532 | Train Loss Unweighted: 0.4812 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.4395 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0205 | Regression_eta: 0.0003 | Regression_pt: 0.0206 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0015 | Classification_binary: 0.4395 | Regression_cos_phi: 0.0641 | Regression_energy: 0.1920 | Regression_eta: 0.0676 | Regression_pt: 0.1994 | Regression_sin_phi: 0.0889 | LR: 4.97e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4618s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0388 | max_optimizer_step: 0.0596 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3299.7 | uss_MB: 2071.6 | child_rss_MB: 22764.4 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:08:57,661] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 03:08:57,727] INFO : Running validation on rank0 +rank0 [2026-11-15 03:08:57,727] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:08:57,727] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:08:57,844] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 03:08:57,949] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:08:58,049] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:08:58,118] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:08:58,184] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:08:58,250] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:08:58,317] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:08:58,384] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:08:58,498] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:08:58,600] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:08:58,701] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:08:58,767] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:08:58,831] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:08:58,894] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:08:58,958] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:08:59,023] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:08:59,121] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:08:59,127] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:08:59,129] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:08:59,130] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:08:59,133] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:08:59,135] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:08:59,136] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:08:59,138] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:08:59,140] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:08:59,140] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:08:59,142] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:08:59,144] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:08:59,145] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:08:59,146] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:08:59,148] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:08:59,151] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:09:00,540] INFO : VALIDATION | Step=24000/500000 | Train Loss=1.0532 | Valid Loss=16.8185 | Train Loss Unweighted=0.4812 | Valid Loss Unweighted=2.0502 | Stale=19 | ETA=936.0m +rank0 [2026-11-15 03:09:00,541] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:09:00,615] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:09:00,619] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:09:00,664] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:09:01,133] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:09:01,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:09:01,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:09:01,874] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:09:02,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:09:02,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:09:02,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:09:02,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:09:03,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:09:03,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:09:03,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:09:03,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:09:04,077] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:09:04,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:09:04,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:09:04,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:09:04,828] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:09:04,834] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:09:05,368] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_24000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:09:05,762] INFO : Plotted number of elements +rank0 [2026-11-15 03:09:07,014] INFO : Plotted jets +rank0 [2026-11-15 03:09:08,377] INFO : Plotted jet ratio +rank0 [2026-11-15 03:09:26,727] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 03:09:26,727] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:10:27,705] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 03:10:27,705] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:10:57,509] INFO : Step 25000/500000 rank0 | Train Loss: 1.9763 | Train Loss Unweighted: 0.5359 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.4049 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0651 | Regression_eta: 0.0004 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.4049 | Regression_cos_phi: 0.1668 | Regression_energy: 0.6085 | Regression_eta: 0.0944 | Regression_pt: 0.6302 | Regression_sin_phi: 0.0715 | LR: 5.22e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5551s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0589 | max_tensorboard_logging: 0.2224 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3325.9 | uss_MB: 2097.8 | child_rss_MB: 22762.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:10:57,511] INFO : Running validation on rank0 +rank0 [2026-11-15 03:10:57,511] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:10:57,511] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:10:57,629] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 03:10:57,733] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:10:57,835] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:10:57,904] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:10:57,971] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:10:58,037] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:10:58,104] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:10:58,170] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:10:58,281] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:10:58,385] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:10:58,488] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:10:58,555] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:10:58,620] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:10:58,684] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:10:58,748] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:10:58,812] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:10:58,910] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:10:58,916] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:10:58,918] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:10:58,920] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:10:58,922] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:10:58,924] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:10:58,926] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:10:58,928] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:10:58,930] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:10:58,931] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:10:58,933] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:10:58,935] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:10:58,936] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:10:58,938] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:10:58,940] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:10:58,941] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:11:00,328] INFO : VALIDATION | Step=25000/500000 | Train Loss=1.9763 | Valid Loss=13.1204 | Train Loss Unweighted=0.5359 | Valid Loss Unweighted=2.2639 | Stale=20 | ETA=934.6m +rank0 [2026-11-15 03:11:00,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:11:00,448] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:11:00,454] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:11:00,495] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:11:00,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:11:01,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:11:01,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:11:01,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:11:01,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:11:02,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:11:02,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:11:02,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:11:02,837] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:11:03,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:11:03,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:11:03,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:11:03,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:11:04,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:11:04,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:11:04,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:11:04,551] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:11:04,556] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:11:05,088] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_25000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:11:05,476] INFO : Plotted number of elements +rank0 [2026-11-15 03:11:06,718] INFO : Plotted jets +rank0 [2026-11-15 03:11:08,059] INFO : Plotted jet ratio +rank0 [2026-11-15 03:11:39,569] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 03:11:39,569] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:12:40,252] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 03:12:40,253] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:12:56,666] INFO : Step 26000/500000 rank0 | Train Loss: 0.5243 | Train Loss Unweighted: 0.0864 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0558 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0154 | Regression_eta: 0.0001 | Regression_pt: 0.0149 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0010 | Classification_binary: 0.0558 | Regression_cos_phi: 0.0874 | Regression_energy: 0.1444 | Regression_eta: 0.0229 | Regression_pt: 0.1435 | Regression_sin_phi: 0.0693 | LR: 5.47e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4727s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0030 | max_backward: 0.0403 | max_optimizer_step: 0.0609 | max_tensorboard_logging: 0.1300 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3338.3 | uss_MB: 2110.1 | child_rss_MB: 22766.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:12:56,668] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 03:12:56,736] INFO : Running validation on rank0 +rank0 [2026-11-15 03:12:56,736] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:12:56,736] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:12:56,857] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 03:12:56,963] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:12:57,063] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:12:57,130] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:12:57,195] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:12:57,259] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:12:57,325] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:12:57,391] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:12:57,504] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:12:57,608] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:12:57,715] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:12:57,782] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:12:57,848] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:12:57,915] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:12:57,982] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:12:58,049] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:12:58,150] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:12:58,156] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:12:58,158] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:12:58,160] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:12:58,162] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:12:58,164] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:12:58,165] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:12:58,167] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:12:58,168] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:12:58,169] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:12:58,171] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:12:58,173] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:12:58,174] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:12:58,177] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:12:58,179] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:12:58,181] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:12:59,588] INFO : VALIDATION | Step=26000/500000 | Train Loss=0.5243 | Valid Loss=18.7163 | Train Loss Unweighted=0.0864 | Valid Loss Unweighted=2.3894 | Stale=21 | ETA=933.0m +rank0 [2026-11-15 03:12:59,589] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:12:59,662] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:12:59,666] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:12:59,712] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:13:00,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:13:00,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:13:00,650] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:13:00,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:13:01,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:13:01,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:13:01,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:13:01,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:13:02,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:13:02,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:13:02,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:13:02,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:13:03,053] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:13:03,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:13:03,526] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:13:03,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:13:03,800] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:13:03,805] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:13:04,344] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_26000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:13:04,748] INFO : Plotted number of elements +rank0 [2026-11-15 03:13:05,986] INFO : Plotted jets +rank0 [2026-11-15 03:13:07,377] INFO : Plotted jet ratio +rank0 [2026-11-15 03:13:52,392] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 03:13:52,393] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:14:52,880] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 03:14:52,881] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:14:55,839] INFO : Step 27000/500000 rank0 | Train Loss: 0.5833 | Train Loss Unweighted: 0.1423 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1182 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0119 | Regression_eta: 0.0002 | Regression_pt: 0.0118 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.1182 | Regression_cos_phi: 0.1112 | Regression_energy: 0.1113 | Regression_eta: 0.0491 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0794 | LR: 5.74e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4578s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0382 | max_optimizer_step: 0.0587 | max_tensorboard_logging: 0.1287 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3372.7 | uss_MB: 2144.6 | child_rss_MB: 22766.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:14:55,840] INFO : Running validation on rank0 +rank0 [2026-11-15 03:14:55,841] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:14:55,841] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:14:55,964] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 03:14:56,074] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:14:56,175] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:14:56,245] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:14:56,311] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:14:56,378] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:14:56,446] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:14:56,513] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:14:56,629] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:14:56,734] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:14:56,835] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:14:56,901] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:14:56,966] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:14:57,030] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:14:57,094] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:14:57,158] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:14:57,257] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:14:57,263] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:14:57,265] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:14:57,267] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:14:57,269] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:14:57,271] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:14:57,272] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:14:57,274] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:14:57,275] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:14:57,276] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:14:57,278] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:14:57,280] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:14:57,280] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:14:57,282] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:14:57,284] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:14:57,286] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:14:58,689] INFO : VALIDATION | Step=27000/500000 | Train Loss=0.5833 | Valid Loss=18.1313 | Train Loss Unweighted=0.1423 | Valid Loss Unweighted=2.3444 | Stale=22 | ETA=931.3m +rank0 [2026-11-15 03:14:58,689] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:14:58,763] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:14:58,767] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:14:58,812] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:14:59,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:14:59,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:14:59,766] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:15:00,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:15:00,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:15:00,473] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:15:00,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:15:00,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:15:01,188] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:15:01,437] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:15:01,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:15:01,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:15:02,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:15:02,383] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:15:02,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:15:02,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:15:02,881] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:15:02,886] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:15:03,425] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_27000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:15:03,816] INFO : Plotted number of elements +rank0 [2026-11-15 03:15:05,065] INFO : Plotted jets +rank0 [2026-11-15 03:15:06,434] INFO : Plotted jet ratio +rank0 [2026-11-15 03:16:04,679] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 03:16:04,679] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:16:54,628] INFO : Step 28000/500000 rank0 | Train Loss: 0.8734 | Train Loss Unweighted: 0.1075 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.0565 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0249 | Regression_eta: 0.0001 | Regression_pt: 0.0254 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0218 | Classification_binary: 0.0565 | Regression_cos_phi: 0.1361 | Regression_energy: 0.2328 | Regression_eta: 0.0252 | Regression_pt: 0.2454 | Regression_sin_phi: 0.1556 | LR: 6.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4700s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0026 | max_backward: 0.0385 | max_optimizer_step: 0.0605 | max_tensorboard_logging: 0.1281 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3395.8 | uss_MB: 2167.7 | child_rss_MB: 22764.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:16:54,630] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 03:16:54,698] INFO : Running validation on rank0 +rank0 [2026-11-15 03:16:54,698] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:16:54,698] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:16:54,816] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 03:16:54,923] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:16:55,024] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:16:55,093] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:16:55,160] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:16:55,226] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:16:55,295] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:16:55,363] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:16:55,476] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:16:55,580] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:16:55,683] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:16:55,750] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:16:55,816] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:16:55,881] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:16:55,947] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:16:56,011] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:16:56,110] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:16:56,116] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:16:56,118] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:16:56,120] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:16:56,122] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:16:56,124] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:16:56,125] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:16:56,125] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:16:56,128] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:16:56,129] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:16:56,130] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:16:56,133] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:16:56,133] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:16:56,135] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:16:56,137] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:16:56,139] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:16:57,546] INFO : VALIDATION | Step=28000/500000 | Train Loss=0.8734 | Valid Loss=16.8242 | Train Loss Unweighted=0.1075 | Valid Loss Unweighted=2.3713 | Stale=23 | ETA=929.6m +rank0 [2026-11-15 03:16:57,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:16:57,619] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:16:57,625] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:16:57,675] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:16:58,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:16:58,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:16:58,646] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:16:58,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:16:59,115] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:16:59,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:16:59,586] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:16:59,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:17:00,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:17:00,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:17:00,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:17:00,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:17:01,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:17:01,249] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:17:01,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:17:01,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:17:01,770] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:17:01,779] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:17:02,311] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_28000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:17:02,700] INFO : Plotted number of elements +rank0 [2026-11-15 03:17:03,933] INFO : Plotted jets +rank0 [2026-11-15 03:17:05,285] INFO : Plotted jet ratio +rank0 [2026-11-15 03:17:16,306] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 03:17:16,306] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:18:16,338] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 03:18:16,338] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:18:52,829] INFO : Step 29000/500000 rank0 | Train Loss: 1.0738 | Train Loss Unweighted: 0.1216 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0371 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0414 | Regression_eta: 0.0003 | Regression_pt: 0.0425 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0002 | Classification_binary: 0.0371 | Regression_cos_phi: 0.0910 | Regression_energy: 0.3872 | Regression_eta: 0.0666 | Regression_pt: 0.4104 | Regression_sin_phi: 0.0812 | LR: 6.29e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4557s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0589 | max_tensorboard_logging: 0.1267 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3413.2 | uss_MB: 2185.1 | child_rss_MB: 22766.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:18:52,831] INFO : Running validation on rank0 +rank0 [2026-11-15 03:18:52,831] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:18:52,831] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:18:52,949] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 03:18:53,054] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:18:53,153] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:18:53,221] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:18:53,287] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:18:53,353] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:18:53,420] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:18:53,487] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:18:53,601] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:18:53,711] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:18:53,819] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:18:53,885] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:18:53,949] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:18:54,013] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:18:54,078] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:18:54,142] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:18:54,240] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:18:54,246] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:18:54,248] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:18:54,250] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:18:54,252] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:18:54,254] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:18:54,255] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:18:54,257] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:18:54,258] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:18:54,259] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:18:54,261] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:18:54,263] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:18:54,265] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:18:54,265] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:18:54,267] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:18:54,271] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:18:55,659] INFO : VALIDATION | Step=29000/500000 | Train Loss=1.0738 | Valid Loss=14.2513 | Train Loss Unweighted=0.1216 | Valid Loss Unweighted=2.5250 | Stale=24 | ETA=927.6m +rank0 [2026-11-15 03:18:55,659] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:18:55,731] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:18:55,735] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:18:55,780] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:18:56,251] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:18:56,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:18:56,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:18:56,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:18:57,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:18:57,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:18:57,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:18:57,910] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:18:58,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:18:58,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:18:58,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:18:58,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:18:59,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:18:59,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:18:59,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:18:59,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:18:59,856] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:18:59,861] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:19:00,403] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_29000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:19:00,791] INFO : Plotted number of elements +rank0 [2026-11-15 03:19:02,040] INFO : Plotted jets +rank0 [2026-11-15 03:19:03,390] INFO : Plotted jet ratio +rank0 [2026-11-15 03:19:27,670] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 03:19:27,670] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:20:27,946] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 03:20:27,947] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:20:51,291] INFO : Step 30000/500000 rank0 | Train Loss: 0.4130 | Train Loss Unweighted: 0.0748 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0491 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0125 | Regression_eta: 0.0002 | Regression_pt: 0.0129 | Regression_sin_phi: 0.0000 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0000 | Classification_binary: 0.0491 | Regression_cos_phi: 0.0444 | Regression_energy: 0.1172 | Regression_eta: 0.0425 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0353 | LR: 6.58e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4973s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0354 | max_loss: 0.0027 | max_backward: 0.0391 | max_optimizer_step: 0.0745 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3436.3 | uss_MB: 2208.2 | child_rss_MB: 22766.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:20:51,293] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 03:20:51,361] INFO : Running validation on rank0 +rank0 [2026-11-15 03:20:51,362] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:20:51,362] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:20:51,478] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 03:20:51,584] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:20:51,687] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:20:51,755] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:20:51,821] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:20:51,887] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:20:51,954] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:20:52,020] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:20:52,136] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:20:52,239] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:20:52,342] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:20:52,410] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:20:52,476] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:20:52,544] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:20:52,611] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:20:52,678] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:20:52,779] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:20:52,785] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:20:52,787] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:20:52,789] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:20:52,791] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:20:52,793] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:20:52,794] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:20:52,796] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:20:52,797] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:20:52,798] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:20:52,800] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:20:52,802] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:20:52,803] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:20:52,804] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:20:52,806] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:20:52,808] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:20:54,197] INFO : VALIDATION | Step=30000/500000 | Train Loss=0.4130 | Valid Loss=18.9530 | Train Loss Unweighted=0.0748 | Valid Loss Unweighted=2.6810 | Stale=25 | ETA=925.7m +rank0 [2026-11-15 03:20:54,197] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:20:54,271] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:20:54,274] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:54,319] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:20:54,768] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:20:55,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:20:55,251] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:20:55,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:20:55,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:20:55,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:20:56,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:20:56,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:20:56,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:20:56,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:20:57,161] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:20:57,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:20:57,645] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:20:57,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:20:58,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:20:58,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:20:58,391] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:20:58,396] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:58,930] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:59,331] INFO : Plotted number of elements +rank0 [2026-11-15 03:21:00,558] INFO : Plotted jets +rank0 [2026-11-15 03:21:01,922] INFO : Plotted jet ratio +rank0 [2026-11-15 03:21:39,529] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 03:21:39,529] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:22:39,989] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 03:22:39,989] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:22:50,015] INFO : Step 31000/500000 rank0 | Train Loss: 0.7005 | Train Loss Unweighted: 0.1567 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1090 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0237 | Regression_eta: 0.0001 | Regression_pt: 0.0237 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.1090 | Regression_cos_phi: 0.0381 | Regression_energy: 0.2215 | Regression_eta: 0.0339 | Regression_pt: 0.2294 | Regression_sin_phi: 0.0686 | LR: 6.88e-05 | DataLoad Time: 0.0004s | Model Forward Time: 0.4822s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0379 | max_loss: 0.0031 | max_backward: 0.0401 | max_optimizer_step: 0.0623 | max_tensorboard_logging: 0.1340 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3392.6 | uss_MB: 2164.5 | child_rss_MB: 22766.2 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:22:50,016] INFO : Running validation on rank0 +rank0 [2026-11-15 03:22:50,016] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:22:50,016] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:22:50,137] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 03:22:50,245] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:22:50,346] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:22:50,416] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:22:50,482] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:22:50,548] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:22:50,614] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:22:50,680] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:22:50,792] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:22:50,903] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:22:51,015] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:22:51,084] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:22:51,152] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:22:51,217] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:22:51,284] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:22:51,352] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:22:51,456] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:22:51,462] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:22:51,464] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:22:51,466] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:22:51,468] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:22:51,470] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:22:51,471] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:22:51,473] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:22:51,473] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:22:51,475] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:22:51,477] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:22:51,479] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:22:51,481] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:22:51,483] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:22:51,483] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:22:51,487] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:22:52,912] INFO : VALIDATION | Step=31000/500000 | Train Loss=0.7005 | Valid Loss=19.4268 | Train Loss Unweighted=0.1567 | Valid Loss Unweighted=2.6205 | Stale=26 | ETA=923.9m +rank0 [2026-11-15 03:22:52,913] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:22:52,987] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:22:52,991] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:53,037] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:22:53,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:22:53,729] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:22:53,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:22:54,222] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:22:54,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:22:54,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:22:54,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:22:55,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:22:55,420] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:22:55,659] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:22:55,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:22:56,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:22:56,387] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:22:56,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:22:56,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:22:57,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:22:57,140] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:22:57,146] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:57,680] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_31000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:58,084] INFO : Plotted number of elements +rank0 [2026-11-15 03:22:59,310] INFO : Plotted jets +rank0 [2026-11-15 03:23:00,681] INFO : Plotted jet ratio +rank0 [2026-11-15 03:23:51,821] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-15 03:23:51,821] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:24:49,037] INFO : Step 32000/500000 rank0 | Train Loss: 0.9360 | Train Loss Unweighted: 0.1525 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1000 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0273 | Regression_eta: 0.0006 | Regression_pt: 0.0244 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0002 | Classification_binary: 0.1000 | Regression_cos_phi: 0.0914 | Regression_energy: 0.2549 | Regression_eta: 0.1608 | Regression_pt: 0.2360 | Regression_sin_phi: 0.0927 | LR: 7.19e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4730s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0028 | max_backward: 0.0395 | max_optimizer_step: 0.0628 | max_tensorboard_logging: 0.1288 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3492.3 | uss_MB: 2264.2 | child_rss_MB: 22765.4 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:24:49,039] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-32000.pth +rank0 [2026-11-15 03:24:49,108] INFO : Running validation on rank0 +rank0 [2026-11-15 03:24:49,108] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:24:49,108] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:24:49,226] INFO : saved batch0_epoch32000.parquet +rank0 [2026-11-15 03:24:49,330] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:24:49,430] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:24:49,496] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:24:49,561] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:24:49,625] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:24:49,691] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:24:49,757] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:24:49,871] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:24:49,982] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:24:50,094] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:24:50,162] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:24:50,229] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:24:50,293] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:24:50,358] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:24:50,423] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:24:50,523] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:24:50,529] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:24:50,531] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:24:50,533] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:24:50,535] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:24:50,537] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:24:50,537] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:24:50,539] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:24:50,541] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:24:50,542] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:24:50,544] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:24:50,544] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:24:50,546] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:24:50,548] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:24:50,550] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:24:50,552] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:24:51,948] INFO : VALIDATION | Step=32000/500000 | Train Loss=0.9360 | Valid Loss=19.7769 | Train Loss Unweighted=0.1525 | Valid Loss Unweighted=2.7567 | Stale=27 | ETA=922.1m +rank0 [2026-11-15 03:24:51,949] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:24:52,022] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:24:52,027] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:52,074] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:24:52,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:24:52,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:24:53,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:24:53,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:24:53,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:24:53,737] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:24:53,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:24:54,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:24:54,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:24:54,690] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:24:54,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:24:55,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:24:55,409] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:24:55,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:24:55,878] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:24:56,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:24:56,159] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:24:56,164] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:56,715] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_32000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:57,111] INFO : Plotted number of elements +rank0 [2026-11-15 03:24:58,365] INFO : Plotted jets +rank0 [2026-11-15 03:24:59,730] INFO : Plotted jet ratio +rank0 [2026-11-15 03:25:03,716] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-15 03:25:03,716] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:26:04,244] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-15 03:26:04,245] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:26:48,005] INFO : Step 33000/500000 rank0 | Train Loss: 0.6567 | Train Loss Unweighted: 0.0826 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0527 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0146 | Regression_eta: 0.0003 | Regression_pt: 0.0147 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0007 | Classification_binary: 0.0527 | Regression_cos_phi: 0.1668 | Regression_energy: 0.1362 | Regression_eta: 0.0693 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0890 | LR: 7.51e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4899s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0394 | max_loss: 0.0030 | max_backward: 0.0408 | max_optimizer_step: 0.0650 | max_tensorboard_logging: 0.1280 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3515.3 | uss_MB: 2287.2 | child_rss_MB: 22763.7 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:26:48,006] INFO : Running validation on rank0 +rank0 [2026-11-15 03:26:48,006] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:26:48,006] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:26:48,125] INFO : saved batch0_epoch33000.parquet +rank0 [2026-11-15 03:26:48,230] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:26:48,329] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:26:48,399] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:26:48,466] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:26:48,532] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:26:48,598] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:26:48,663] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:26:48,776] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:26:48,882] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:26:48,988] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:26:49,056] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:26:49,123] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:26:49,189] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:26:49,255] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:26:49,319] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:26:49,421] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:26:49,427] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:26:49,429] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:26:49,431] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:26:49,433] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:26:49,435] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:26:49,436] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:26:49,438] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:26:49,440] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:26:49,440] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:26:49,442] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:26:49,444] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:26:49,446] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:26:49,448] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:26:49,450] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:26:49,452] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:26:50,848] INFO : VALIDATION | Step=33000/500000 | Train Loss=0.6567 | Valid Loss=16.3618 | Train Loss Unweighted=0.0826 | Valid Loss Unweighted=2.9069 | Stale=28 | ETA=920.3m +rank0 [2026-11-15 03:26:50,849] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:26:50,923] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:26:50,927] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:50,974] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:26:51,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:26:51,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:26:51,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:26:52,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:26:52,398] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:26:52,635] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:26:52,870] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:26:53,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:26:53,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:26:53,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:26:53,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:26:54,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:26:54,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:26:54,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:26:54,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:26:54,983] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:26:55,057] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:26:55,062] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:55,597] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_33000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:56,001] INFO : Plotted number of elements +rank0 [2026-11-15 03:26:57,234] INFO : Plotted jets +rank0 [2026-11-15 03:26:58,607] INFO : Plotted jet ratio +rank0 [2026-11-15 03:27:15,898] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-15 03:27:15,898] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:28:16,059] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-15 03:28:16,059] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:28:46,468] INFO : Step 34000/500000 rank0 | Train Loss: 1.1343 | Train Loss Unweighted: 0.3672 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.2947 | Regression_cos_phi: 0.0000 | Regression_energy: 0.0360 | Regression_eta: 0.0002 | Regression_pt: 0.0361 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.2947 | Regression_cos_phi: 0.0233 | Regression_energy: 0.3364 | Regression_eta: 0.0460 | Regression_pt: 0.3487 | Regression_sin_phi: 0.0850 | LR: 7.83e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4764s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0384 | max_optimizer_step: 0.0608 | max_tensorboard_logging: 0.1349 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3466.6 | uss_MB: 2238.5 | child_rss_MB: 22766.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:28:46,470] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/checkpoints/checkpoint-34000.pth +rank0 [2026-11-15 03:28:46,537] INFO : Running validation on rank0 +rank0 [2026-11-15 03:28:46,537] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:28:46,537] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:28:46,661] INFO : saved batch0_epoch34000.parquet +rank0 [2026-11-15 03:28:46,771] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:28:46,875] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:28:46,944] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:28:47,010] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:28:47,076] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:28:47,142] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:28:47,208] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:28:47,321] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:28:47,431] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:28:47,541] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:28:47,607] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:28:47,672] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:28:47,737] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:28:47,802] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:28:47,867] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:28:47,970] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:28:47,976] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:28:47,978] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:28:47,980] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:28:47,982] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:28:47,984] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:28:47,985] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:28:47,986] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:28:47,989] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:28:47,989] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:28:47,991] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:28:47,992] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:28:47,994] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:28:47,996] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:28:47,996] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:28:48,000] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:28:49,429] INFO : VALIDATION | Step=34000/500000 | Train Loss=1.1343 | Valid Loss=14.8704 | Train Loss Unweighted=0.3672 | Valid Loss Unweighted=2.8519 | Stale=29 | ETA=918.4m +rank0 [2026-11-15 03:28:49,429] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:28:49,505] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:28:49,508] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:49,559] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:28:50,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:28:50,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:28:50,544] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:28:50,789] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:28:51,028] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:28:51,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:28:51,510] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:28:51,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:28:51,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:28:52,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:28:52,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:28:52,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:28:52,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:28:53,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:28:53,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:28:53,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:28:53,712] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:28:53,718] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:54,245] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_34000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:54,646] INFO : Plotted number of elements +rank0 [2026-11-15 03:28:55,889] INFO : Plotted jets +rank0 [2026-11-15 03:28:57,247] INFO : Plotted jet ratio +rank0 [2026-11-15 03:29:27,798] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-15 03:29:27,799] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:30:28,618] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-15 03:30:28,618] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:30:45,701] INFO : Step 35000/500000 rank0 | Train Loss: 0.4871 | Train Loss Unweighted: 0.1599 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1391 | Regression_cos_phi: 0.0000 | Regression_energy: 0.0105 | Regression_eta: 0.0003 | Regression_pt: 0.0099 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 83.8698 | Classification_binary: 1.0000 | Regression_cos_phi: 725.5200 | Regression_energy: 9.3485 | Regression_eta: 253.4228 | Regression_pt: 9.6582 | Regression_sin_phi: 796.6372 | Task Weighted Losses: Classification: 0.0002 | Classification_binary: 0.1391 | Regression_cos_phi: 0.0314 | Regression_energy: 0.0985 | Regression_eta: 0.0742 | Regression_pt: 0.0953 | Regression_sin_phi: 0.0484 | LR: 8.16e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4638s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0601 | max_tensorboard_logging: 0.1293 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3567.4 | uss_MB: 2339.3 | child_rss_MB: 22766.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:30:45,703] INFO : Running validation on rank0 +rank0 [2026-11-15 03:30:45,703] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:30:45,703] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:30:45,823] INFO : saved batch0_epoch35000.parquet +rank0 [2026-11-15 03:30:45,927] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:30:46,028] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:30:46,098] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:30:46,164] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:30:46,231] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:30:46,299] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:30:46,368] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:30:46,483] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:30:46,596] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:30:46,706] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:30:46,773] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:30:46,839] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:30:46,902] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:30:46,967] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:30:47,032] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:30:47,135] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:30:47,141] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:30:47,143] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:30:47,145] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:30:47,147] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:30:47,149] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:30:47,150] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:30:47,151] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:30:47,153] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:30:47,155] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:30:47,156] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:30:47,158] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:30:47,159] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:30:47,161] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:30:47,163] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:30:47,165] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:30:48,579] INFO : VALIDATION | Step=35000/500000 | Train Loss=0.4871 | Valid Loss=19.2367 | Train Loss Unweighted=0.1599 | Valid Loss Unweighted=3.1031 | Stale=30 | ETA=916.6m +rank0 [2026-11-15 03:30:48,579] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:30:48,654] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:30:48,657] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:48,705] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:30:49,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:30:49,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:30:49,675] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:30:49,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:30:50,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:30:50,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:30:50,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:30:50,869] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:30:51,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:30:51,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:30:51,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:30:51,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:30:52,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:30:52,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:30:52,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:30:52,755] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:30:52,803] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:30:52,809] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:53,348] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_2p5mm_seed12345_20261115_022112_689707/preds_step_35000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:53,741] INFO : Plotted number of elements +rank0 [2026-11-15 03:30:54,983] INFO : Plotted jets +rank0 [2026-11-15 03:30:56,334] INFO : Plotted jet ratio +rank0 [2026-11-15 03:30:56,675] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 03:30:56,675] INFO : Training completed. Total time on device 0: 69.130min diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..b9b0dfb36ba213024e5691b5bd129cd6ceba6916 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:431a5057c9667ce3a6fec1a07578b5d05a3d61401379c4425e1cd0ada7d9905d +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-32000.pth b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-32000.pth new file mode 100644 index 0000000000000000000000000000000000000000..6ac4d4cde9931a7506a887e3e92e98daad92df5f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-32000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea793e8c236905c664cf6c2f66641b074519dbde0e5af263210855ecf88a4db4 +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_1000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_1000.json new file mode 100644 index 0000000000000000000000000000000000000000..963299ff261a545903da4422b814a684eb4753b6 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_1000.json @@ -0,0 +1 @@ +{"epoch": 1.7793594306049823, "train": {"Classification_binary": 1.932424783706665, "Classification": 0.0027650948613882065, "Regression_pt": 0.08539735525846481, "Regression_eta": 0.0004510619619395584, "Regression_sin_phi": 0.0007957819616422057, "Regression_cos_phi": 0.0004936522454954684, "Regression_energy": 0.08443247526884079, "Total": 5.268239498138428, "Total_unweighted": 2.106760205264436}, "train_group": {}, "valid": {"Classification_binary": 1.0488148927688599, "Classification": 0.009072788059711456, "Regression_pt": 0.09484686702489853, "Regression_eta": 0.0027877071406692266, "Regression_sin_phi": 0.0008743288344703615, "Regression_cos_phi": 0.0008531691855750978, "Regression_energy": 0.09220152348279953, "Total": 6.4279961585998535, "Total_unweighted": 1.249451276496984, "metrics/particle/class_1/efficiency": 0.9665272235870361, "metrics/particle/class_1/pid_accuracy": 0.9675324559211731, "metrics/particle/class_2/efficiency": 0.5714285969734192, "metrics/particle/class_2/pid_accuracy": 0.875, "metrics/particle/class_3/efficiency": 0.8549323081970215, "metrics/particle/class_3/pid_accuracy": 0.9954751133918762, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.9468085169792175, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.05999999865889549, "metrics/particle/count/mae": 0.2919999957084656, "metrics/particle/count/prediction_mean": 4.748000144958496, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.04844885692000389, "metrics/particle/event/energy_response_mean": 0.9958770275115967, "metrics/particle/event/met_abs_error": 1.3354231119155884, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0479256696999073, "metrics/particle/event/scalar_pt_response_mean": 0.9947137236595154, "metrics/particle/event/vector_pt_closure": 0.04942428693175316, "metrics/particle/matched/delta_eta_abs_mean": 0.008435540832579136, "metrics/particle/matched/delta_phi_abs_mean": 0.007724658586084843, "metrics/particle/matched/delta_r_mean": 0.012842808850109577, "metrics/particle/matched/energy_relative_abs_mean": 0.06209712475538254, "metrics/particle/matched/pid_accuracy": 0.9781221747398376, "metrics/particle/matched/pt_relative_abs_mean": 0.06246551126241684, "metrics/particle/matching/duplicate_fraction": 0.015164279378950596, "metrics/particle/matching/efficiency": 0.9126455783843994, "metrics/particle/matching/f1": 0.9183759093284607, "metrics/particle/matching/purity": 0.9241786003112793, "metrics/particle/matching/target_coverage_dr0p05": 0.897670567035675, "metrics/particle/matching/target_coverage_dr0p10": 0.9392678737640381, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.837770402431488, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9251247644424438}, "step_train_time": 0.8129167556762695, "step_valid_time": 2.057167489998392, "step_total_time": 74.5162250995636, "elapsed_training_loop_seconds": 74.5162250995636, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0023204823513319, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.037859261610986183, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 438, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9797752808988764, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9954337899543378, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9875424688561721, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0045662100456621, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9797752808988764, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9954337899543378, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9875424688561721, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9797752808988764, "step_test_time": 3.393594959998154, "step_plot_time": 3.330342037996161, "step_validation_cycle_time": 9.266000391013222} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..d436b59563afd80f5164bbb3be8cec23f082198c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 0.5214858651161194, "Classification": 0.013263767585158348, "Regression_pt": 0.029181430116295815, "Regression_eta": 0.0008261156617663801, "Regression_sin_phi": 0.00033729529241099954, "Regression_cos_phi": 0.0005654218839481473, "Regression_energy": 0.03110908903181553, "Total": 3.269735813140869, "Total_unweighted": 0.5967689846875146}, "train_group": {}, "valid": {"Classification_binary": 0.9865744113922119, "Classification": 0.00857430137693882, "Regression_pt": 0.09217478334903717, "Regression_eta": 0.0027552135288715363, "Regression_sin_phi": 0.001073819468729198, "Regression_cos_phi": 0.0011573272058740258, "Regression_energy": 0.09056504815816879, "Total": 6.577624797821045, "Total_unweighted": 1.1828749044798315, "metrics/particle/class_1/efficiency": 0.9665272235870361, "metrics/particle/class_1/pid_accuracy": 0.9826839566230774, "metrics/particle/class_2/efficiency": 0.5714285969734192, "metrics/particle/class_2/pid_accuracy": 0.875, "metrics/particle/class_3/efficiency": 0.8742746710777283, "metrics/particle/class_3/pid_accuracy": 0.997787594795227, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.936170220375061, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.03999999910593033, "metrics/particle/count/mae": 0.2639999985694885, "metrics/particle/count/prediction_mean": 4.76800012588501, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.045028768479824066, "metrics/particle/event/energy_response_mean": 0.9887561798095703, "metrics/particle/event/met_abs_error": 1.254772663116455, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04525597020983696, "metrics/particle/event/scalar_pt_response_mean": 0.9878994226455688, "metrics/particle/event/vector_pt_closure": 0.04571545869112015, "metrics/particle/matched/delta_eta_abs_mean": 0.006761922966688871, "metrics/particle/matched/delta_phi_abs_mean": 0.007608889136463404, "metrics/particle/matched/delta_r_mean": 0.011416834779083729, "metrics/particle/matched/energy_relative_abs_mean": 0.060186419636011124, "metrics/particle/matched/pid_accuracy": 0.9846431612968445, "metrics/particle/matched/pt_relative_abs_mean": 0.06008737534284592, "metrics/particle/matching/duplicate_fraction": 0.015100670978426933, "metrics/particle/matching/efficiency": 0.9209650754928589, "metrics/particle/matching/f1": 0.9248120188713074, "metrics/particle/matching/purity": 0.9286912679672241, "metrics/particle/matching/target_coverage_dr0p05": 0.9109817147254944, "metrics/particle/matching/target_coverage_dr0p10": 0.941763699054718, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8452578783035278, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9342762231826782}, "step_train_time": 0.8395564556121826, "step_valid_time": 1.8980622829985805, "step_total_time": 796.2354490756989, "elapsed_training_loop_seconds": 796.2354490756989, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9990072216757676, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02894966126251397, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 438, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9775280898876404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9931506849315068, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9852774631936579, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.00684931506849315, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9775280898876404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9931506849315068, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9852774631936579, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9775280898876404, "step_test_time": 3.0200352990068495, "step_plot_time": 2.6979616259923205, "step_validation_cycle_time": 8.033104620990343} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_11000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_11000.json new file mode 100644 index 0000000000000000000000000000000000000000..8f1d28f820ee69522a907955db56f16000eef2a8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_11000.json @@ -0,0 +1 @@ +{"epoch": 19.572953736654803, "train": {"Classification_binary": 0.4837706685066223, "Classification": 0.00018525138148106635, "Regression_pt": 0.03518860787153244, "Regression_eta": 0.00019707743194885552, "Regression_sin_phi": 0.0011266450164839625, "Regression_cos_phi": 0.0005277590244077146, "Regression_energy": 0.0343475416302681, "Total": 2.702498435974121, "Total_unweighted": 0.5553435508627445}, "train_group": {}, "valid": {"Classification_binary": 0.9749444127082825, "Classification": 0.008982221595942974, "Regression_pt": 0.0875450000166893, "Regression_eta": 0.002758890390396118, "Regression_sin_phi": 0.0012102052569389343, "Regression_cos_phi": 0.0014465010026469827, "Regression_energy": 0.08614792674779892, "Total": 6.750143527984619, "Total_unweighted": 1.1630351577186957, "metrics/particle/class_1/efficiency": 0.9686192274093628, "metrics/particle/class_1/pid_accuracy": 0.9784017205238342, "metrics/particle/class_2/efficiency": 0.6071428656578064, "metrics/particle/class_2/pid_accuracy": 0.8235294222831726, "metrics/particle/class_3/efficiency": 0.8762089014053345, "metrics/particle/class_3/pid_accuracy": 0.9977924823760986, "metrics/particle/class_4/efficiency": 0.96875, "metrics/particle/class_4/pid_accuracy": 0.9569892287254333, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.03999999910593033, "metrics/particle/count/mae": 0.2720000147819519, "metrics/particle/count/prediction_mean": 4.76800012588501, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.04652189090847969, "metrics/particle/event/energy_response_mean": 0.9922716021537781, "metrics/particle/event/met_abs_error": 1.379781723022461, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04589000716805458, "metrics/particle/event/scalar_pt_response_mean": 0.9921417832374573, "metrics/particle/event/vector_pt_closure": 0.046902142465114594, "metrics/particle/matched/delta_eta_abs_mean": 0.007366691250354052, "metrics/particle/matched/delta_phi_abs_mean": 0.00795789435505867, "metrics/particle/matched/delta_r_mean": 0.012098180130124092, "metrics/particle/matched/energy_relative_abs_mean": 0.058307357132434845, "metrics/particle/matched/pid_accuracy": 0.9837691783905029, "metrics/particle/matched/pt_relative_abs_mean": 0.05844021961092949, "metrics/particle/matching/duplicate_fraction": 0.011744966730475426, "metrics/particle/matching/efficiency": 0.9226289391517639, "metrics/particle/matching/f1": 0.9264828562736511, "metrics/particle/matching/purity": 0.9303691387176514, "metrics/particle/matching/target_coverage_dr0p05": 0.9101497530937195, "metrics/particle/matching/target_coverage_dr0p10": 0.9450914859771729, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8510815501213074, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9334442615509033}, "step_train_time": 0.8443779945373535, "step_valid_time": 1.8725495450053131, "step_total_time": 876.7831130027771, "elapsed_training_loop_seconds": 876.7831130027771, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0008222849838142, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.029270957600862735, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 440, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9797752808988764, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.990909090909091, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9853107344632769, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.00909090909090909, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9752808988764045, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9863636363636363, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9807909604519774, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9797752808988764, "step_test_time": 2.9774432419944787, "step_plot_time": 2.6127895819954574, "step_validation_cycle_time": 7.820470527993166} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_12000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_12000.json new file mode 100644 index 0000000000000000000000000000000000000000..7418fd38c5fa0b738336d036fd51b43bee97453a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_12000.json @@ -0,0 +1 @@ +{"epoch": 21.352313167259787, "train": {"Classification_binary": 0.6897686719894409, "Classification": 4.652144343708642e-05, "Regression_pt": 0.011700725182890892, "Regression_eta": 0.00030724576208740473, "Regression_sin_phi": 0.0003626630932558328, "Regression_cos_phi": 0.0002874166239053011, "Regression_energy": 0.010186158120632172, "Total": 1.5575112104415894, "Total_unweighted": 0.7126594022156496}, "train_group": {}, "valid": {"Classification_binary": 0.9965495467185974, "Classification": 0.010666841641068459, "Regression_pt": 0.09165138751268387, "Regression_eta": 0.002864224137738347, "Regression_sin_phi": 0.0011864053085446358, "Regression_cos_phi": 0.0018350903410464525, "Regression_energy": 0.09073712676763535, "Total": 7.189290523529053, "Total_unweighted": 1.1954906224273145, "metrics/particle/class_1/efficiency": 0.9665272235870361, "metrics/particle/class_1/pid_accuracy": 0.9826839566230774, "metrics/particle/class_2/efficiency": 0.5357142686843872, "metrics/particle/class_2/pid_accuracy": 0.8666666746139526, "metrics/particle/class_3/efficiency": 0.8781431317329407, "metrics/particle/class_3/pid_accuracy": 0.9911894202232361, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.957446813583374, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9759036302566528, "metrics/particle/count/bias_mean": -0.004000000189989805, "metrics/particle/count/mae": 0.2759999930858612, "metrics/particle/count/prediction_mean": 4.803999900817871, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.048350315541028976, "metrics/particle/event/energy_response_mean": 0.9926257133483887, "metrics/particle/event/met_abs_error": 1.432538628578186, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04875816032290459, "metrics/particle/event/scalar_pt_response_mean": 0.9925125241279602, "metrics/particle/event/vector_pt_closure": 0.04879620671272278, "metrics/particle/matched/delta_eta_abs_mean": 0.007074846886098385, "metrics/particle/matched/delta_phi_abs_mean": 0.006913716904819012, "metrics/particle/matched/delta_r_mean": 0.011031006462872028, "metrics/particle/matched/energy_relative_abs_mean": 0.060247406363487244, "metrics/particle/matched/pid_accuracy": 0.9819494485855103, "metrics/particle/matched/pt_relative_abs_mean": 0.060328688472509384, "metrics/particle/matching/duplicate_fraction": 0.01498751062899828, "metrics/particle/matching/efficiency": 0.921796977519989, "metrics/particle/matching/f1": 0.9221805930137634, "metrics/particle/matching/purity": 0.9225645065307617, "metrics/particle/matching/target_coverage_dr0p05": 0.9076539278030396, "metrics/particle/matching/target_coverage_dr0p10": 0.9442595839500427, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8485856652259827, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9351081252098083}, "step_train_time": 0.7712399959564209, "step_valid_time": 1.8982833639893215, "step_total_time": 956.4994337558746, "elapsed_training_loop_seconds": 956.4994337558746, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000857223982866, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0312820019232487, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 443, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 440, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 439, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9887640449438202, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9932279909706546, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.990990990990991, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.006772009029345372, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9865168539325843, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9909706546275395, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9887387387387387, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9887640449438202, "step_test_time": 3.015476617991226, "step_plot_time": 2.700895653004409, "step_validation_cycle_time": 8.039108882003347} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_13000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_13000.json new file mode 100644 index 0000000000000000000000000000000000000000..f68d06a3d3fdf57d9f3047beb931ed484d8c8fcc --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_13000.json @@ -0,0 +1 @@ +{"epoch": 23.131672597864767, "train": {"Classification_binary": 1.3921802043914795, "Classification": 0.0006913150427863002, "Regression_pt": 0.023356594145298004, "Regression_eta": 0.0002688080130610615, "Regression_sin_phi": 0.00012520182644948363, "Regression_cos_phi": 0.00045173661783337593, "Regression_energy": 0.02290588989853859, "Total": 2.3959856033325195, "Total_unweighted": 1.4399797499354463}, "train_group": {}, "valid": {"Classification_binary": 1.040895938873291, "Classification": 0.010904825292527676, "Regression_pt": 0.08588068932294846, "Regression_eta": 0.0028757278341799974, "Regression_sin_phi": 0.0011281217448413372, "Regression_cos_phi": 0.0014718641759827733, "Regression_energy": 0.08550062030553818, "Total": 6.936263084411621, "Total_unweighted": 1.2286577875493094, "metrics/particle/class_1/efficiency": 0.9665272235870361, "metrics/particle/class_1/pid_accuracy": 0.9805194735527039, "metrics/particle/class_2/efficiency": 0.5714285969734192, "metrics/particle/class_2/pid_accuracy": 0.8125, "metrics/particle/class_3/efficiency": 0.8781431317329407, "metrics/particle/class_3/pid_accuracy": 0.9955947399139404, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.9468085169792175, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.01600000075995922, "metrics/particle/count/mae": 0.2800000011920929, "metrics/particle/count/prediction_mean": 4.791999816894531, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.04695620760321617, "metrics/particle/event/energy_response_mean": 0.9958587288856506, "metrics/particle/event/met_abs_error": 1.431347370147705, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04672880098223686, "metrics/particle/event/scalar_pt_response_mean": 0.9956858158111572, "metrics/particle/event/vector_pt_closure": 0.046478331089019775, "metrics/particle/matched/delta_eta_abs_mean": 0.007387993857264519, "metrics/particle/matched/delta_phi_abs_mean": 0.0068022157065570354, "metrics/particle/matched/delta_r_mean": 0.011189903132617474, "metrics/particle/matched/energy_relative_abs_mean": 0.06001758575439453, "metrics/particle/matched/pid_accuracy": 0.982867419719696, "metrics/particle/matched/pt_relative_abs_mean": 0.060115426778793335, "metrics/particle/matching/duplicate_fraction": 0.015025041997432709, "metrics/particle/matching/efficiency": 0.9226289391517639, "metrics/particle/matching/f1": 0.9241666793823242, "metrics/particle/matching/purity": 0.9257095456123352, "metrics/particle/matching/target_coverage_dr0p05": 0.9043261408805847, "metrics/particle/matching/target_coverage_dr0p10": 0.9434276223182678, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8485856652259827, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9367720484733582}, "step_train_time": 0.7444496154785156, "step_valid_time": 1.8663709209940862, "step_total_time": 1037.2015578746796, "elapsed_training_loop_seconds": 1037.2015578746796, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001881358065432, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.032365935134995194, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 441, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9797752808988764, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9886621315192744, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9841986455981941, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.011337868480725623, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9775280898876404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9863945578231292, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.981941309255079, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9797752808988764, "step_test_time": 2.90675186199951, "step_plot_time": 2.609753466997063, "step_validation_cycle_time": 7.735437579001882} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_14000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_14000.json new file mode 100644 index 0000000000000000000000000000000000000000..422f0a6e574f507997f71e25fb0fd373c5367e0a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_14000.json @@ -0,0 +1 @@ +{"epoch": 24.91103202846975, "train": {"Classification_binary": 0.39212509989738464, "Classification": 0.0001212081842822954, "Regression_pt": 0.07725262641906738, "Regression_eta": 0.00019797042477875948, "Regression_sin_phi": 0.0003623583179432899, "Regression_cos_phi": 0.0002930753689724952, "Regression_energy": 0.07661586254835129, "Total": 2.694028377532959, "Total_unweighted": 0.5469682011607802}, "train_group": {}, "valid": {"Classification_binary": 1.0139223337173462, "Classification": 0.012104236520826817, "Regression_pt": 0.08683089911937714, "Regression_eta": 0.0029415518511086702, "Regression_sin_phi": 0.001123812166042626, "Regression_cos_phi": 0.0019179348601028323, "Regression_energy": 0.08573444932699203, "Total": 7.225017547607422, "Total_unweighted": 1.2045752175617963, "metrics/particle/class_1/efficiency": 0.9707112908363342, "metrics/particle/class_1/pid_accuracy": 0.9806034564971924, "metrics/particle/class_2/efficiency": 0.5714285969734192, "metrics/particle/class_2/pid_accuracy": 0.875, "metrics/particle/class_3/efficiency": 0.8839458227157593, "metrics/particle/class_3/pid_accuracy": 0.9978117942810059, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.957446813583374, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9638554453849792, "metrics/particle/count/bias_mean": 0.0, "metrics/particle/count/mae": 0.29600000381469727, "metrics/particle/count/prediction_mean": 4.808000087738037, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.04682940989732742, "metrics/particle/event/energy_response_mean": 0.9950913786888123, "metrics/particle/event/met_abs_error": 1.3677186965942383, "metrics/particle/event/scalar_pt_relative_abs_error": 0.046888627111911774, "metrics/particle/event/scalar_pt_response_mean": 0.9946843981742859, "metrics/particle/event/vector_pt_closure": 0.04748321324586868, "metrics/particle/matched/delta_eta_abs_mean": 0.0074805631302297115, "metrics/particle/matched/delta_phi_abs_mean": 0.007610894739627838, "metrics/particle/matched/delta_r_mean": 0.011897116899490356, "metrics/particle/matched/energy_relative_abs_mean": 0.06117026135325432, "metrics/particle/matched/pid_accuracy": 0.9829443693161011, "metrics/particle/matched/pt_relative_abs_mean": 0.061284150928258896, "metrics/particle/matching/duplicate_fraction": 0.01747088134288788, "metrics/particle/matching/efficiency": 0.9267886877059937, "metrics/particle/matching/f1": 0.9267886877059937, "metrics/particle/matching/purity": 0.9267886877059937, "metrics/particle/matching/target_coverage_dr0p05": 0.9068219661712646, "metrics/particle/matching/target_coverage_dr0p10": 0.9450914859771729, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8494176268577576, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9384359121322632}, "step_train_time": 0.7714757919311523, "step_valid_time": 1.9068010479968507, "step_total_time": 1117.719351530075, "elapsed_training_loop_seconds": 1117.719351530075, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0007251759880273, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.029656407368890858, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 441, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 438, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9842696629213483, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9931972789115646, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9887133182844243, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.006802721088435374, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9797752808988764, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9886621315192744, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9841986455981941, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9842696629213483, "step_test_time": 3.016529618995264, "step_plot_time": 2.717005190992495, "step_validation_cycle_time": 8.05435036199924} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_15000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_15000.json new file mode 100644 index 0000000000000000000000000000000000000000..335d6f92da5eae81681f4947c84e4aacb05f5655 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_15000.json @@ -0,0 +1 @@ +{"epoch": 26.690391459074732, "train": {"Classification_binary": 0.7602782249450684, "Classification": 2.6948615413857624e-05, "Regression_pt": 0.02371841110289097, "Regression_eta": 0.0001267326297238469, "Regression_sin_phi": 0.00018620234914124012, "Regression_cos_phi": 0.0005437547224573791, "Regression_energy": 0.02318524196743965, "Total": 1.7449805736541748, "Total_unweighted": 0.8080655163321353}, "train_group": {}, "valid": {"Classification_binary": 1.0941797494888306, "Classification": 0.01202428713440895, "Regression_pt": 0.09071338921785355, "Regression_eta": 0.002853894606232643, "Regression_sin_phi": 0.0012030507205054164, "Regression_cos_phi": 0.0016891603590920568, "Regression_energy": 0.09074835479259491, "Total": 7.346928119659424, "Total_unweighted": 1.293411886319518, "metrics/particle/class_1/efficiency": 0.9644351601600647, "metrics/particle/class_1/pid_accuracy": 0.9804772138595581, "metrics/particle/class_2/efficiency": 0.5714285969734192, "metrics/particle/class_2/pid_accuracy": 0.875, "metrics/particle/class_3/efficiency": 0.8839458227157593, "metrics/particle/class_3/pid_accuracy": 0.9978117942810059, "metrics/particle/class_4/efficiency": 0.96875, "metrics/particle/class_4/pid_accuracy": 0.9462365508079529, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9759036302566528, "metrics/particle/count/bias_mean": 0.00800000037997961, "metrics/particle/count/mae": 0.2800000011920929, "metrics/particle/count/prediction_mean": 4.815999984741211, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.047219663858413696, "metrics/particle/event/energy_response_mean": 0.9978445172309875, "metrics/particle/event/met_abs_error": 1.3896903991699219, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04721035435795784, "metrics/particle/event/scalar_pt_response_mean": 0.9972424507141113, "metrics/particle/event/vector_pt_closure": 0.04718821868300438, "metrics/particle/matched/delta_eta_abs_mean": 0.006697190925478935, "metrics/particle/matched/delta_phi_abs_mean": 0.006723012309521437, "metrics/particle/matched/delta_r_mean": 0.010652320459485054, "metrics/particle/matched/energy_relative_abs_mean": 0.06065377593040466, "metrics/particle/matched/pid_accuracy": 0.9828828573226929, "metrics/particle/matched/pt_relative_abs_mean": 0.0608011893928051, "metrics/particle/matching/duplicate_fraction": 0.016611294820904732, "metrics/particle/matching/efficiency": 0.9234609007835388, "metrics/particle/matching/f1": 0.9226932525634766, "metrics/particle/matching/purity": 0.9219269156455994, "metrics/particle/matching/target_coverage_dr0p05": 0.9101497530937195, "metrics/particle/matching/target_coverage_dr0p10": 0.9450914859771729, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8494176268577576, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9376040101051331}, "step_train_time": 0.7524330615997314, "step_valid_time": 1.8799690560117597, "step_total_time": 1198.2945384979248, "elapsed_training_loop_seconds": 1198.2945384979248, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0030689108862334, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.030603145339719307, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 440, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 438, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 437, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9842696629213483, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9954545454545455, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9898305084745763, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.004545454545454545, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9820224719101124, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9931818181818182, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9875706214689266, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9842696629213483, "step_test_time": 2.948280030992464, "step_plot_time": 2.6430546379997395, "step_validation_cycle_time": 7.827110433005146} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_16000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_16000.json new file mode 100644 index 0000000000000000000000000000000000000000..2d0c2e3294eb744be799d409016a1d5b1d8b31a0 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_16000.json @@ -0,0 +1 @@ +{"epoch": 28.469750889679716, "train": {"Classification_binary": 0.7414982318878174, "Classification": 0.0007048144470900297, "Regression_pt": 0.10356874018907547, "Regression_eta": 0.0006969990208745003, "Regression_sin_phi": 0.0001890740095404908, "Regression_cos_phi": 0.0009349897736683488, "Regression_energy": 0.1019597202539444, "Total": 3.9819788932800293, "Total_unweighted": 0.9495525695820106}, "train_group": {}, "valid": {"Classification_binary": 1.0990841388702393, "Classification": 0.013715044595301151, "Regression_pt": 0.09199661761522293, "Regression_eta": 0.0027475631795823574, "Regression_sin_phi": 0.0011725460644811392, "Regression_cos_phi": 0.0017371114809066057, "Regression_energy": 0.09218603372573853, "Total": 7.464553356170654, "Total_unweighted": 1.302639055531472, "metrics/particle/class_1/efficiency": 0.9623430967330933, "metrics/particle/class_1/pid_accuracy": 0.980434775352478, "metrics/particle/class_2/efficiency": 0.6071428656578064, "metrics/particle/class_2/pid_accuracy": 0.8823529481887817, "metrics/particle/class_3/efficiency": 0.8839458227157593, "metrics/particle/class_3/pid_accuracy": 0.9956236481666565, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.9255319237709045, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9759036302566528, "metrics/particle/count/bias_mean": 0.012000000104308128, "metrics/particle/count/mae": 0.2759999930858612, "metrics/particle/count/prediction_mean": 4.820000171661377, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.04623344913125038, "metrics/particle/event/energy_response_mean": 0.9982264637947083, "metrics/particle/event/met_abs_error": 1.4027069807052612, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04655325040221214, "metrics/particle/event/scalar_pt_response_mean": 0.9977993369102478, "metrics/particle/event/vector_pt_closure": 0.04648508504033089, "metrics/particle/matched/delta_eta_abs_mean": 0.00726816663518548, "metrics/particle/matched/delta_phi_abs_mean": 0.007281915284693241, "metrics/particle/matched/delta_r_mean": 0.011453627608716488, "metrics/particle/matched/energy_relative_abs_mean": 0.06101766601204872, "metrics/particle/matched/pid_accuracy": 0.9801980257034302, "metrics/particle/matched/pt_relative_abs_mean": 0.06105594336986542, "metrics/particle/matching/duplicate_fraction": 0.01825726218521595, "metrics/particle/matching/efficiency": 0.9242928624153137, "metrics/particle/matching/f1": 0.923140823841095, "metrics/particle/matching/purity": 0.9219917058944702, "metrics/particle/matching/target_coverage_dr0p05": 0.9143095016479492, "metrics/particle/matching/target_coverage_dr0p10": 0.9475873708724976, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.841930091381073, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9367720484733582}, "step_train_time": 0.8026764392852783, "step_valid_time": 1.8793687959987437, "step_total_time": 1279.2136862277985, "elapsed_training_loop_seconds": 1279.2136862277985, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0008943461413158, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.03170581616734136, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 440, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 438, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 437, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9842696629213483, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9954545454545455, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9898305084745763, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.004545454545454545, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9820224719101124, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9931818181818182, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9875706214689266, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9842696629213483, "step_test_time": 2.9948630559956655, "step_plot_time": 2.710963081000955, "step_validation_cycle_time": 8.002152588000172} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_17000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_17000.json new file mode 100644 index 0000000000000000000000000000000000000000..54188782c3d9f09a74caeae0321b01364941649f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_17000.json @@ -0,0 +1 @@ +{"epoch": 30.249110320284696, "train": {"Classification_binary": 0.680509090423584, "Classification": 0.00041405268711969256, "Regression_pt": 0.011569309048354626, "Regression_eta": 0.00041153389611281455, "Regression_sin_phi": 0.00025569656281732023, "Regression_cos_phi": 0.00010118392674485222, "Regression_energy": 0.012653619050979614, "Total": 1.474613070487976, "Total_unweighted": 0.7059144855957129}, "train_group": {}, "valid": {"Classification_binary": 1.1682623624801636, "Classification": 0.014646141789853573, "Regression_pt": 0.08684065192937851, "Regression_eta": 0.002774161286652088, "Regression_sin_phi": 0.0012467359192669392, "Regression_cos_phi": 0.001709353644400835, "Regression_energy": 0.08673440665006638, "Total": 7.570163726806641, "Total_unweighted": 1.362213813699782, "metrics/particle/class_1/efficiency": 0.9644351601600647, "metrics/particle/class_1/pid_accuracy": 0.9783080220222473, "metrics/particle/class_2/efficiency": 0.5714285969734192, "metrics/particle/class_2/pid_accuracy": 0.9375, "metrics/particle/class_3/efficiency": 0.8820115923881531, "metrics/particle/class_3/pid_accuracy": 0.9956140518188477, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.9468085169792175, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9759036302566528, "metrics/particle/count/bias_mean": 0.01600000075995922, "metrics/particle/count/mae": 0.30399999022483826, "metrics/particle/count/prediction_mean": 4.823999881744385, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.05106774717569351, "metrics/particle/event/energy_response_mean": 0.9938993453979492, "metrics/particle/event/met_abs_error": 1.4830009937286377, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0506257601082325, "metrics/particle/event/scalar_pt_response_mean": 0.9928135275840759, "metrics/particle/event/vector_pt_closure": 0.05017488822340965, "metrics/particle/matched/delta_eta_abs_mean": 0.007173454854637384, "metrics/particle/matched/delta_phi_abs_mean": 0.006582660600543022, "metrics/particle/matched/delta_r_mean": 0.010873968712985516, "metrics/particle/matched/energy_relative_abs_mean": 0.062424201518297195, "metrics/particle/matched/pid_accuracy": 0.9819819927215576, "metrics/particle/matched/pt_relative_abs_mean": 0.06257195770740509, "metrics/particle/matching/duplicate_fraction": 0.017412934452295303, "metrics/particle/matching/efficiency": 0.9234609007835388, "metrics/particle/matching/f1": 0.9219269156455994, "metrics/particle/matching/purity": 0.9203979969024658, "metrics/particle/matching/target_coverage_dr0p05": 0.9076539278030396, "metrics/particle/matching/target_coverage_dr0p10": 0.9459234476089478, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.837770402431488, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9376040101051331}, "step_train_time": 0.8140320777893066, "step_valid_time": 1.8666090649930993, "step_total_time": 1359.9517567157745, "elapsed_training_loop_seconds": 1359.9517567157745, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0003542370214078, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.030653627486868507, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 439, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9797752808988764, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9931662870159453, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9864253393665159, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.00683371298405467, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9775280898876404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9908883826879271, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9841628959276018, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9797752808988764, "step_test_time": 2.9919356249884004, "step_plot_time": 2.8030477379943477, "step_validation_cycle_time": 8.137403115993948} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_18000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_18000.json new file mode 100644 index 0000000000000000000000000000000000000000..a9defc5800693bd5e34c6048d656ae9388a1d053 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_18000.json @@ -0,0 +1 @@ +{"epoch": 32.02846975088968, "train": {"Classification_binary": 0.36558207869529724, "Classification": 0.00021209804981481284, "Regression_pt": 0.03327954560518265, "Regression_eta": 0.00024359689268749207, "Regression_sin_phi": 7.101701339706779e-05, "Regression_cos_phi": 0.00018148620438296348, "Regression_energy": 0.03218717873096466, "Total": 1.3724130392074585, "Total_unweighted": 0.4317570011917269}, "train_group": {}, "valid": {"Classification_binary": 1.2393208742141724, "Classification": 0.015234374441206455, "Regression_pt": 0.08478894084692001, "Regression_eta": 0.002780535724014044, "Regression_sin_phi": 0.0012123861815780401, "Regression_cos_phi": 0.0015771054895594716, "Regression_energy": 0.08517301827669144, "Total": 7.567913055419922, "Total_unweighted": 1.4300872351741418, "metrics/particle/class_1/efficiency": 0.9644351601600647, "metrics/particle/class_1/pid_accuracy": 0.9783080220222473, "metrics/particle/class_2/efficiency": 0.5, "metrics/particle/class_2/pid_accuracy": 0.9285714030265808, "metrics/particle/class_3/efficiency": 0.8781431317329407, "metrics/particle/class_3/pid_accuracy": 0.9911894202232361, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.9255319237709045, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9759036302566528, "metrics/particle/count/bias_mean": 0.00800000037997961, "metrics/particle/count/mae": 0.31200000643730164, "metrics/particle/count/prediction_mean": 4.815999984741211, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.048738230019807816, "metrics/particle/event/energy_response_mean": 0.9897087216377258, "metrics/particle/event/met_abs_error": 1.4086418151855469, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04804923012852669, "metrics/particle/event/scalar_pt_response_mean": 0.9883643388748169, "metrics/particle/event/vector_pt_closure": 0.04860466718673706, "metrics/particle/matched/delta_eta_abs_mean": 0.006339368410408497, "metrics/particle/matched/delta_phi_abs_mean": 0.006806012708693743, "metrics/particle/matched/delta_r_mean": 0.010308045893907547, "metrics/particle/matched/energy_relative_abs_mean": 0.06175968796014786, "metrics/particle/matched/pid_accuracy": 0.978300154209137, "metrics/particle/matched/pt_relative_abs_mean": 0.061820901930332184, "metrics/particle/matching/duplicate_fraction": 0.01910299062728882, "metrics/particle/matching/efficiency": 0.920133113861084, "metrics/particle/matching/f1": 0.9193682670593262, "metrics/particle/matching/purity": 0.9186046719551086, "metrics/particle/matching/target_coverage_dr0p05": 0.9109817147254944, "metrics/particle/matching/target_coverage_dr0p10": 0.941763699054718, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8394342660903931, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9351081252098083}, "step_train_time": 0.8256359100341797, "step_valid_time": 1.8757792369870003, "step_total_time": 1443.326205253601, "elapsed_training_loop_seconds": 1443.326205253601, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9971950425656714, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.028150429173952052, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 440, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 438, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9842696629213483, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9954545454545455, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9898305084745763, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.004545454545454545, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9775280898876404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9886363636363636, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9830508474576272, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9842696629213483, "step_test_time": 3.061246787008713, "step_plot_time": 2.7252193710010033, "step_validation_cycle_time": 8.080812992004212} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_19000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_19000.json new file mode 100644 index 0000000000000000000000000000000000000000..37a5f3773e0e6664af3206cb1a7957abac321d23 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_19000.json @@ -0,0 +1 @@ +{"epoch": 33.80782918149466, "train": {"Classification_binary": 0.4634377360343933, "Classification": 2.7410815164330415e-05, "Regression_pt": 0.02968519553542137, "Regression_eta": 0.0001472996809752658, "Regression_sin_phi": 8.288795652333647e-05, "Regression_cos_phi": 0.00012249113933648914, "Regression_energy": 0.02948012575507164, "Total": 1.3276400566101074, "Total_unweighted": 0.5229831469168857}, "train_group": {}, "valid": {"Classification_binary": 1.2533519268035889, "Classification": 0.016452906653285027, "Regression_pt": 0.085173599421978, "Regression_eta": 0.003095116699114442, "Regression_sin_phi": 0.0012476886622607708, "Regression_cos_phi": 0.0019325382309034467, "Regression_energy": 0.08641345053911209, "Total": 8.024831771850586, "Total_unweighted": 1.4476672270102426, "metrics/particle/class_1/efficiency": 0.9623430967330933, "metrics/particle/class_1/pid_accuracy": 0.980434775352478, "metrics/particle/class_2/efficiency": 0.5714285969734192, "metrics/particle/class_2/pid_accuracy": 0.875, "metrics/particle/class_3/efficiency": 0.8800773620605469, "metrics/particle/class_3/pid_accuracy": 0.997802197933197, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.936170220375061, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9759036302566528, "metrics/particle/count/bias_mean": 0.035999998450279236, "metrics/particle/count/mae": 0.30799999833106995, "metrics/particle/count/prediction_mean": 4.843999862670898, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.047736309468746185, "metrics/particle/event/energy_response_mean": 0.9958767890930176, "metrics/particle/event/met_abs_error": 1.3788740634918213, "metrics/particle/event/scalar_pt_relative_abs_error": 0.047345682978630066, "metrics/particle/event/scalar_pt_response_mean": 0.9950549602508545, "metrics/particle/event/vector_pt_closure": 0.04628545790910721, "metrics/particle/matched/delta_eta_abs_mean": 0.006794547662138939, "metrics/particle/matched/delta_phi_abs_mean": 0.006623117718845606, "metrics/particle/matched/delta_r_mean": 0.010616080835461617, "metrics/particle/matched/energy_relative_abs_mean": 0.0603397861123085, "metrics/particle/matched/pid_accuracy": 0.9819494485855103, "metrics/particle/matched/pt_relative_abs_mean": 0.0604805164039135, "metrics/particle/matching/duplicate_fraction": 0.01899256743490696, "metrics/particle/matching/efficiency": 0.921796977519989, "metrics/particle/matching/f1": 0.9183588624000549, "metrics/particle/matching/purity": 0.9149463176727295, "metrics/particle/matching/target_coverage_dr0p05": 0.9134775400161743, "metrics/particle/matching/target_coverage_dr0p10": 0.9450914859771729, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8477537631988525, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9359400868415833}, "step_train_time": 0.8897278308868408, "step_valid_time": 1.9125686729967128, "step_total_time": 1522.710769891739, "elapsed_training_loop_seconds": 1522.710769891739, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.999694045315128, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.026044351455158155, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 444, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 439, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 437, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9865168539325843, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9887387387387387, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9876265466816648, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.01126126126126126, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9820224719101124, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9842342342342343, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.983127109111361, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9865168539325843, "step_test_time": 3.061816084009479, "step_plot_time": 2.752241230002255, "step_validation_cycle_time": 8.190998732010485} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_2000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_2000.json new file mode 100644 index 0000000000000000000000000000000000000000..76d3c8fc9e6400f3c7594f671668f929153d692f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_2000.json @@ -0,0 +1 @@ +{"epoch": 3.5587188612099645, "train": {"Classification_binary": 0.7749474048614502, "Classification": 0.008129751309752464, "Regression_pt": 0.08461550623178482, "Regression_eta": 0.0002704006910789758, "Regression_sin_phi": 0.00014362786896526814, "Regression_cos_phi": 0.00017358333570882678, "Regression_energy": 0.08155601471662521, "Total": 3.6654582023620605, "Total_unweighted": 0.9498362890153658}, "train_group": {}, "valid": {"Classification_binary": 1.0192641019821167, "Classification": 0.008130471222102642, "Regression_pt": 0.08887002617120743, "Regression_eta": 0.002802337519824505, "Regression_sin_phi": 0.0008649517549201846, "Regression_cos_phi": 0.0008206361089833081, "Regression_energy": 0.08598124980926514, "Total": 6.166551113128662, "Total_unweighted": 1.20673377456842, "metrics/particle/class_1/efficiency": 0.9707112908363342, "metrics/particle/class_1/pid_accuracy": 0.9719827771186829, "metrics/particle/class_2/efficiency": 0.5714285969734192, "metrics/particle/class_2/pid_accuracy": 0.875, "metrics/particle/class_3/efficiency": 0.8626692295074463, "metrics/particle/class_3/pid_accuracy": 0.9932735562324524, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.9468085169792175, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.04399999976158142, "metrics/particle/count/mae": 0.2759999930858612, "metrics/particle/count/prediction_mean": 4.763999938964844, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.0487196110188961, "metrics/particle/event/energy_response_mean": 0.9974187612533569, "metrics/particle/event/met_abs_error": 1.3119014501571655, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04769643768668175, "metrics/particle/event/scalar_pt_response_mean": 0.9962557554244995, "metrics/particle/event/vector_pt_closure": 0.04883873835206032, "metrics/particle/matched/delta_eta_abs_mean": 0.007806009147316217, "metrics/particle/matched/delta_phi_abs_mean": 0.007323327474296093, "metrics/particle/matched/delta_r_mean": 0.01201827172189951, "metrics/particle/matched/energy_relative_abs_mean": 0.061937347054481506, "metrics/particle/matched/pid_accuracy": 0.9791477918624878, "metrics/particle/matched/pt_relative_abs_mean": 0.06209779903292656, "metrics/particle/matching/duplicate_fraction": 0.01679261215031147, "metrics/particle/matching/efficiency": 0.917637288570404, "metrics/particle/matching/f1": 0.9218553900718689, "metrics/particle/matching/purity": 0.9261125326156616, "metrics/particle/matching/target_coverage_dr0p05": 0.9059900045394897, "metrics/particle/matching/target_coverage_dr0p10": 0.9442595839500427, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8494176268577576, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9301164746284485}, "step_train_time": 0.7687268257141113, "step_valid_time": 1.8483461790019646, "step_total_time": 154.2438395023346, "elapsed_training_loop_seconds": 154.2438395023346, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000272561087328, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.033441078153590764, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 437, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9797752808988764, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9977116704805492, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9886621315192744, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.002288329519450801, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9752808988764045, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9931350114416476, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9841269841269841, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9797752808988764, "step_test_time": 2.949000528999022, "step_plot_time": 2.6379606900009094, "step_validation_cycle_time": 7.869370713000535} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..5d7984ef90c775840e663abaf1dc8605f011b259 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 0.6321814060211182, "Classification": 0.0002118531265296042, "Regression_pt": 0.01835216023027897, "Regression_eta": 0.00019281897402834147, "Regression_sin_phi": 0.0001236316020367667, "Regression_cos_phi": 0.00012441261787898839, "Regression_energy": 0.018671151250600815, "Total": 1.3269822597503662, "Total_unweighted": 0.6698574338224716}, "train_group": {}, "valid": {"Classification_binary": 1.3346302509307861, "Classification": 0.01710691675543785, "Regression_pt": 0.09544914960861206, "Regression_eta": 0.0030986617784947157, "Regression_sin_phi": 0.0011721363989636302, "Regression_cos_phi": 0.002047307090833783, "Regression_energy": 0.09628432244062424, "Total": 8.358158111572266, "Total_unweighted": 1.5497887450037524, "metrics/particle/class_1/efficiency": 0.9602510333061218, "metrics/particle/class_1/pid_accuracy": 0.984749436378479, "metrics/particle/class_2/efficiency": 0.5, "metrics/particle/class_2/pid_accuracy": 0.9285714030265808, "metrics/particle/class_3/efficiency": 0.8704062104225159, "metrics/particle/class_3/pid_accuracy": 0.9933333396911621, "metrics/particle/class_4/efficiency": 0.96875, "metrics/particle/class_4/pid_accuracy": 0.9247311949729919, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9879518151283264, "metrics/particle/count/bias_mean": 0.004000000189989805, "metrics/particle/count/mae": 0.3240000009536743, "metrics/particle/count/prediction_mean": 4.811999797821045, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.049617163836956024, "metrics/particle/event/energy_response_mean": 0.9900520443916321, "metrics/particle/event/met_abs_error": 1.3860095739364624, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04930353909730911, "metrics/particle/event/scalar_pt_response_mean": 0.9895412921905518, "metrics/particle/event/vector_pt_closure": 0.05035027489066124, "metrics/particle/matched/delta_eta_abs_mean": 0.007252953480929136, "metrics/particle/matched/delta_phi_abs_mean": 0.006769965402781963, "metrics/particle/matched/delta_r_mean": 0.011068601161241531, "metrics/particle/matched/energy_relative_abs_mean": 0.05963468551635742, "metrics/particle/matched/pid_accuracy": 0.9827115535736084, "metrics/particle/matched/pt_relative_abs_mean": 0.05945679545402527, "metrics/particle/matching/duplicate_fraction": 0.021612634882330894, "metrics/particle/matching/efficiency": 0.9143095016479492, "metrics/particle/matching/f1": 0.9139293432235718, "metrics/particle/matching/purity": 0.9135494828224182, "metrics/particle/matching/target_coverage_dr0p05": 0.9051580429077148, "metrics/particle/matching/target_coverage_dr0p10": 0.9376040101051331, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.841930091381073, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9284525513648987}, "step_train_time": 0.8906960487365723, "step_valid_time": 2.06886457199289, "step_total_time": 1604.6851572990417, "elapsed_training_loop_seconds": 1604.6851572990417, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0009297230415926, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.028050618411163408, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 439, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9797752808988764, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9931662870159453, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9864253393665159, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.00683371298405467, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9752808988764045, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9886104783599089, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9819004524886877, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9797752808988764, "step_test_time": 3.231116334005492, "step_plot_time": 2.734572940011276, "step_validation_cycle_time": 8.467106258001877} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_21000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_21000.json new file mode 100644 index 0000000000000000000000000000000000000000..589e64838fb3a209d449284443d311904b460589 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_21000.json @@ -0,0 +1 @@ +{"epoch": 37.36654804270463, "train": {"Classification_binary": 1.9020073413848877, "Classification": 0.00010956481855828315, "Regression_pt": 0.10822651535272598, "Regression_eta": 0.001397282350808382, "Regression_sin_phi": 0.00013434146239887923, "Regression_cos_phi": 0.0002975610550493002, "Regression_energy": 0.1000177264213562, "Total": 5.148494720458984, "Total_unweighted": 2.1121903328457847}, "train_group": {}, "valid": {"Classification_binary": 1.4061555862426758, "Classification": 0.018125111237168312, "Regression_pt": 0.09691808372735977, "Regression_eta": 0.0028493902646005154, "Regression_sin_phi": 0.0013268629554659128, "Regression_cos_phi": 0.001983234891667962, "Regression_energy": 0.09746020287275314, "Total": 8.559004783630371, "Total_unweighted": 1.6248184721916914, "metrics/particle/class_1/efficiency": 0.9602510333061218, "metrics/particle/class_1/pid_accuracy": 0.9825708270072937, "metrics/particle/class_2/efficiency": 0.5357142686843872, "metrics/particle/class_2/pid_accuracy": 0.9333333373069763, "metrics/particle/class_3/efficiency": 0.8723404407501221, "metrics/particle/class_3/pid_accuracy": 0.9955654144287109, "metrics/particle/class_4/efficiency": 0.96875, "metrics/particle/class_4/pid_accuracy": 0.9354838728904724, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9759036302566528, "metrics/particle/count/bias_mean": -0.004000000189989805, "metrics/particle/count/mae": 0.3319999873638153, "metrics/particle/count/prediction_mean": 4.803999900817871, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.05120924115180969, "metrics/particle/event/energy_response_mean": 0.9908573031425476, "metrics/particle/event/met_abs_error": 1.4942402839660645, "metrics/particle/event/scalar_pt_relative_abs_error": 0.051260482519865036, "metrics/particle/event/scalar_pt_response_mean": 0.9891870021820068, "metrics/particle/event/vector_pt_closure": 0.051626913249492645, "metrics/particle/matched/delta_eta_abs_mean": 0.006490047089755535, "metrics/particle/matched/delta_phi_abs_mean": 0.006687941495329142, "metrics/particle/matched/delta_r_mean": 0.010364074259996414, "metrics/particle/matched/energy_relative_abs_mean": 0.060275472700595856, "metrics/particle/matched/pid_accuracy": 0.982742965221405, "metrics/particle/matched/pt_relative_abs_mean": 0.06053956598043442, "metrics/particle/matching/duplicate_fraction": 0.01998334750533104, "metrics/particle/matching/efficiency": 0.9159733653068542, "metrics/particle/matching/f1": 0.9163545370101929, "metrics/particle/matching/purity": 0.9167360663414001, "metrics/particle/matching/target_coverage_dr0p05": 0.9059900045394897, "metrics/particle/matching/target_coverage_dr0p10": 0.9384359121322632, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8394342660903931, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9317803382873535}, "step_train_time": 0.9285600185394287, "step_valid_time": 2.0526760119973915, "step_total_time": 1686.1822719573975, "elapsed_training_loop_seconds": 1686.1822719573975, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0004987366700946, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02959795969657042, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 441, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 437, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9820224719101124, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9909297052154195, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9864559819413092, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.009070294784580499, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9752808988764045, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9841269841269841, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9796839729119639, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9820224719101124, "step_test_time": 3.223089566003182, "step_plot_time": 3.1476636590086855, "step_validation_cycle_time": 8.917889247008134} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_22000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_22000.json new file mode 100644 index 0000000000000000000000000000000000000000..a2d585528a6bcd006deff168b1b3fe65d55fabe1 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_22000.json @@ -0,0 +1 @@ +{"epoch": 39.145907473309606, "train": {"Classification_binary": 0.6607555150985718, "Classification": 0.0007782297325320542, "Regression_pt": 0.04198906570672989, "Regression_eta": 0.00015516206622123718, "Regression_sin_phi": 8.486417937092483e-05, "Regression_cos_phi": 0.0002909434260800481, "Regression_energy": 0.04096033796668053, "Total": 1.9264920949935913, "Total_unweighted": 0.7450141181761865}, "train_group": {}, "valid": {"Classification_binary": 1.4459724426269531, "Classification": 0.01829971931874752, "Regression_pt": 0.0954272523522377, "Regression_eta": 0.0030446224845945835, "Regression_sin_phi": 0.0011617529671639204, "Regression_cos_phi": 0.0017494562780484557, "Regression_energy": 0.09522218257188797, "Total": 8.407404899597168, "Total_unweighted": 1.6608774285996333, "metrics/particle/class_1/efficiency": 0.9602510333061218, "metrics/particle/class_1/pid_accuracy": 0.984749436378479, "metrics/particle/class_2/efficiency": 0.5, "metrics/particle/class_2/pid_accuracy": 0.9285714030265808, "metrics/particle/class_3/efficiency": 0.8742746710777283, "metrics/particle/class_3/pid_accuracy": 0.9933628439903259, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.914893627166748, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9879518151283264, "metrics/particle/count/bias_mean": 0.004000000189989805, "metrics/particle/count/mae": 0.3240000009536743, "metrics/particle/count/prediction_mean": 4.811999797821045, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.050175122916698456, "metrics/particle/event/energy_response_mean": 0.9973642826080322, "metrics/particle/event/met_abs_error": 1.5409621000289917, "metrics/particle/event/scalar_pt_relative_abs_error": 0.049893878400325775, "metrics/particle/event/scalar_pt_response_mean": 0.9962238669395447, "metrics/particle/event/vector_pt_closure": 0.05180122330784798, "metrics/particle/matched/delta_eta_abs_mean": 0.007100851275026798, "metrics/particle/matched/delta_phi_abs_mean": 0.007036295719444752, "metrics/particle/matched/delta_r_mean": 0.011083305813372135, "metrics/particle/matched/energy_relative_abs_mean": 0.062060944736003876, "metrics/particle/matched/pid_accuracy": 0.9818511605262756, "metrics/particle/matched/pt_relative_abs_mean": 0.06203017756342888, "metrics/particle/matching/duplicate_fraction": 0.019118869677186012, "metrics/particle/matching/efficiency": 0.9168053269386292, "metrics/particle/matching/f1": 0.9164240956306458, "metrics/particle/matching/purity": 0.916043221950531, "metrics/particle/matching/target_coverage_dr0p05": 0.9076539278030396, "metrics/particle/matching/target_coverage_dr0p10": 0.940099835395813, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.837770402431488, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9334442615509033}, "step_train_time": 0.8013734817504883, "step_valid_time": 1.8689477290026844, "step_total_time": 1768.9636483192444, "elapsed_training_loop_seconds": 1768.9636483192444, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0017097778797661, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.029986528187395556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 440, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 437, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9820224719101124, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9931818181818182, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9875706214689266, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.006818181818181818, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9752808988764045, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9863636363636363, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9807909604519774, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9820224719101124, "step_test_time": 2.9442396749946056, "step_plot_time": 2.6473737889900804, "step_validation_cycle_time": 7.818914481002139} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_23000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_23000.json new file mode 100644 index 0000000000000000000000000000000000000000..2252a1bf13276671b6e55804b35aa0186ae6b29f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_23000.json @@ -0,0 +1 @@ +{"epoch": 40.92526690391459, "train": {"Classification_binary": 0.45703190565109253, "Classification": 6.442786252591759e-05, "Regression_pt": 0.03390240669250488, "Regression_eta": 0.0003361169365234673, "Regression_sin_phi": 0.00016451155534014106, "Regression_cos_phi": 0.00018401312991045415, "Regression_energy": 0.03291557356715202, "Total": 1.604224443435669, "Total_unweighted": 0.5245989553950494}, "train_group": {}, "valid": {"Classification_binary": 1.5608934164047241, "Classification": 0.018850769847631454, "Regression_pt": 0.10061729699373245, "Regression_eta": 0.0033355476334691048, "Regression_sin_phi": 0.0011916599469259381, "Regression_cos_phi": 0.0017954049399122596, "Regression_energy": 0.10205645859241486, "Total": 8.887650489807129, "Total_unweighted": 1.7887405543588102, "metrics/particle/class_1/efficiency": 0.9581589698791504, "metrics/particle/class_1/pid_accuracy": 0.9781659245491028, "metrics/particle/class_2/efficiency": 0.5357142686843872, "metrics/particle/class_2/pid_accuracy": 0.8666666746139526, "metrics/particle/class_3/efficiency": 0.8646034598350525, "metrics/particle/class_3/pid_accuracy": 0.9932885766029358, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.9255319237709045, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9879518151283264, "metrics/particle/count/bias_mean": 0.0, "metrics/particle/count/mae": 0.35199999809265137, "metrics/particle/count/prediction_mean": 4.808000087738037, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.04831548035144806, "metrics/particle/event/energy_response_mean": 0.988093912601471, "metrics/particle/event/met_abs_error": 1.4196281433105469, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04775405302643776, "metrics/particle/event/scalar_pt_response_mean": 0.9874570369720459, "metrics/particle/event/vector_pt_closure": 0.04906564578413963, "metrics/particle/matched/delta_eta_abs_mean": 0.007274331524968147, "metrics/particle/matched/delta_phi_abs_mean": 0.006927718874067068, "metrics/particle/matched/delta_r_mean": 0.011247318238019943, "metrics/particle/matched/energy_relative_abs_mean": 0.06000685691833496, "metrics/particle/matched/pid_accuracy": 0.9790337085723877, "metrics/particle/matched/pt_relative_abs_mean": 0.06013577803969383, "metrics/particle/matching/duplicate_fraction": 0.02246256172657013, "metrics/particle/matching/efficiency": 0.9126455783843994, "metrics/particle/matching/f1": 0.9126455783843994, "metrics/particle/matching/purity": 0.9126455783843994, "metrics/particle/matching/target_coverage_dr0p05": 0.9034941792488098, "metrics/particle/matching/target_coverage_dr0p10": 0.9334442615509033, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.837770402431488, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9301164746284485}, "step_train_time": 0.8050596714019775, "step_valid_time": 1.890699630996096, "step_total_time": 1848.0211799144745, "elapsed_training_loop_seconds": 1848.0211799144745, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9968255375488382, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.027200633031003063, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 441, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 433, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9797752808988764, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9886621315192744, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9841986455981941, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.011337868480725623, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9730337078651685, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.981859410430839, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9774266365688488, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9797752808988764, "step_test_time": 2.996011996991001, "step_plot_time": 2.727897803997621, "step_validation_cycle_time": 8.044229145001736} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_24000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_24000.json new file mode 100644 index 0000000000000000000000000000000000000000..0b82b29353b9c9507bacada82fd8d34870262a33 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_24000.json @@ -0,0 +1 @@ +{"epoch": 42.704626334519574, "train": {"Classification_binary": 0.4408667981624603, "Classification": 0.0003819068078882992, "Regression_pt": 0.018011178821325302, "Regression_eta": 0.0003632562584243715, "Regression_sin_phi": 0.0001940944930538535, "Regression_cos_phi": 0.00012988226080778986, "Regression_energy": 0.0179829653352499, "Total": 1.2911338806152344, "Total_unweighted": 0.47793008213920984}, "train_group": {}, "valid": {"Classification_binary": 1.620956540107727, "Classification": 0.022383270785212517, "Regression_pt": 0.09597594290971756, "Regression_eta": 0.0029947259463369846, "Regression_sin_phi": 0.0012584750074893236, "Regression_cos_phi": 0.0020859548822045326, "Regression_energy": 0.09656774997711182, "Total": 9.155488967895508, "Total_unweighted": 1.8422226596157998, "metrics/particle/class_1/efficiency": 0.9623430967330933, "metrics/particle/class_1/pid_accuracy": 0.9826086759567261, "metrics/particle/class_2/efficiency": 0.5, "metrics/particle/class_2/pid_accuracy": 0.8571428656578064, "metrics/particle/class_3/efficiency": 0.8684719800949097, "metrics/particle/class_3/pid_accuracy": 0.993318498134613, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.936170220375061, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9759036302566528, "metrics/particle/count/bias_mean": 0.019999999552965164, "metrics/particle/count/mae": 0.3400000035762787, "metrics/particle/count/prediction_mean": 4.828000068664551, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.04651348292827606, "metrics/particle/event/energy_response_mean": 0.9983818531036377, "metrics/particle/event/met_abs_error": 1.442107081413269, "metrics/particle/event/scalar_pt_relative_abs_error": 0.046313196420669556, "metrics/particle/event/scalar_pt_response_mean": 0.9980155229568481, "metrics/particle/event/vector_pt_closure": 0.047568585723638535, "metrics/particle/matched/delta_eta_abs_mean": 0.00694663729518652, "metrics/particle/matched/delta_phi_abs_mean": 0.006900572218000889, "metrics/particle/matched/delta_r_mean": 0.01095170620828867, "metrics/particle/matched/energy_relative_abs_mean": 0.05981897562742233, "metrics/particle/matched/pid_accuracy": 0.9809091091156006, "metrics/particle/matched/pt_relative_abs_mean": 0.05942869558930397, "metrics/particle/matching/duplicate_fraction": 0.01657000742852688, "metrics/particle/matching/efficiency": 0.9151414036750793, "metrics/particle/matching/f1": 0.913241982460022, "metrics/particle/matching/purity": 0.911350429058075, "metrics/particle/matching/target_coverage_dr0p05": 0.9068219661712646, "metrics/particle/matching/target_coverage_dr0p10": 0.9367720484733582, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.840266227722168, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9309484362602234}, "step_train_time": 0.7888627052307129, "step_valid_time": 1.895696791994851, "step_total_time": 1927.5947346687317, "elapsed_training_loop_seconds": 1927.5947346687317, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0004050495162438, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.029785631953333058, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 442, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 437, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9820224719101124, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9886877828054299, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9853438556933484, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.011312217194570135, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9752808988764045, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9819004524886877, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9785794813979707, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9820224719101124, "step_test_time": 3.078836629007128, "step_plot_time": 2.6408183430030476, "step_validation_cycle_time": 7.976507785002468} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_25000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_25000.json new file mode 100644 index 0000000000000000000000000000000000000000..0f5c3787eebcd562e2e58dbefe856b27ad965007 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_25000.json @@ -0,0 +1 @@ +{"epoch": 44.48398576512456, "train": {"Classification_binary": 0.22510921955108643, "Classification": 9.005987521959469e-05, "Regression_pt": 0.02915782667696476, "Regression_eta": 0.00012060248263878748, "Regression_sin_phi": 9.129593672696501e-05, "Regression_cos_phi": 0.0001579958770889789, "Regression_energy": 0.028092993423342705, "Total": 1.0830905437469482, "Total_unweighted": 0.2828199938230682}, "train_group": {}, "valid": {"Classification_binary": 1.7682335376739502, "Classification": 0.024594996124505997, "Regression_pt": 0.09519622474908829, "Regression_eta": 0.0029493493493646383, "Regression_sin_phi": 0.0012015734100714326, "Regression_cos_phi": 0.0016166756395250559, "Regression_energy": 0.09545570611953735, "Total": 9.209037780761719, "Total_unweighted": 1.989248063066043, "metrics/particle/class_1/efficiency": 0.9602510333061218, "metrics/particle/class_1/pid_accuracy": 0.9825708270072937, "metrics/particle/class_2/efficiency": 0.6071428656578064, "metrics/particle/class_2/pid_accuracy": 0.8823529481887817, "metrics/particle/class_3/efficiency": 0.8626692295074463, "metrics/particle/class_3/pid_accuracy": 0.9955157041549683, "metrics/particle/class_4/efficiency": 0.9583333134651184, "metrics/particle/class_4/pid_accuracy": 0.945652186870575, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9759036302566528, "metrics/particle/count/bias_mean": 0.04399999976158142, "metrics/particle/count/mae": 0.36399999260902405, "metrics/particle/count/prediction_mean": 4.8520002365112305, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.05118391290307045, "metrics/particle/event/energy_response_mean": 0.9967966079711914, "metrics/particle/event/met_abs_error": 1.5403245687484741, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05138787999749184, "metrics/particle/event/scalar_pt_response_mean": 0.9960238933563232, "metrics/particle/event/vector_pt_closure": 0.052070461213588715, "metrics/particle/matched/delta_eta_abs_mean": 0.00668187253177166, "metrics/particle/matched/delta_phi_abs_mean": 0.0066843521781265736, "metrics/particle/matched/delta_r_mean": 0.010565152391791344, "metrics/particle/matched/energy_relative_abs_mean": 0.061385300010442734, "metrics/particle/matched/pid_accuracy": 0.9826800227165222, "metrics/particle/matched/pt_relative_abs_mean": 0.06151258945465088, "metrics/particle/matching/duplicate_fraction": 0.020610056817531586, "metrics/particle/matching/efficiency": 0.9126455783843994, "metrics/particle/matching/f1": 0.9084886312484741, "metrics/particle/matching/purity": 0.9043693542480469, "metrics/particle/matching/target_coverage_dr0p05": 0.9084858298301697, "metrics/particle/matching/target_coverage_dr0p10": 0.9384359121322632, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8336106538772583, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9292845129966736}, "step_train_time": 0.8366451263427734, "step_valid_time": 1.8950712339865277, "step_total_time": 2008.7636783123016, "elapsed_training_loop_seconds": 2008.7636783123016, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9989889720657561, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.029827497495208855, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 442, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 438, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9842696629213483, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9909502262443439, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9875986471251409, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.00904977375565611, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9775280898876404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9841628959276018, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9808342728297632, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9842696629213483, "step_test_time": 3.1457894740015035, "step_plot_time": 2.7381501350027975, "step_validation_cycle_time": 8.226172788999975} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_26000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_26000.json new file mode 100644 index 0000000000000000000000000000000000000000..67d37dad31f32c97ee8e6dfd6ffb0e4288f70b09 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_26000.json @@ -0,0 +1 @@ +{"epoch": 46.263345195729535, "train": {"Classification_binary": 0.039785053580999374, "Classification": 2.2149973233354103e-07, "Regression_pt": 0.006855812389403582, "Regression_eta": 0.00010242754797218367, "Regression_sin_phi": 4.558813452604227e-05, "Regression_cos_phi": 0.0001235127856489271, "Regression_energy": 0.007317929528653622, "Total": 0.3400460481643677, "Total_unweighted": 0.054230545466936064}, "train_group": {}, "valid": {"Classification_binary": 1.8108024597167969, "Classification": 0.02322712540626526, "Regression_pt": 0.09692616015672684, "Regression_eta": 0.003084158292040229, "Regression_sin_phi": 0.001143549452535808, "Regression_cos_phi": 0.001871951506473124, "Regression_energy": 0.09687044471502304, "Total": 9.27495288848877, "Total_unweighted": 2.033925849245861, "metrics/particle/class_1/efficiency": 0.9623430967330933, "metrics/particle/class_1/pid_accuracy": 0.9717391133308411, "metrics/particle/class_2/efficiency": 0.6428571343421936, "metrics/particle/class_2/pid_accuracy": 0.8888888955116272, "metrics/particle/class_3/efficiency": 0.8704062104225159, "metrics/particle/class_3/pid_accuracy": 0.9933333396911621, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.9255319237709045, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": 0.03200000151991844, "metrics/particle/count/mae": 0.35199999809265137, "metrics/particle/count/prediction_mean": 4.840000152587891, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.049178652465343475, "metrics/particle/event/energy_response_mean": 0.9910899996757507, "metrics/particle/event/met_abs_error": 1.4442421197891235, "metrics/particle/event/scalar_pt_relative_abs_error": 0.048571791499853134, "metrics/particle/event/scalar_pt_response_mean": 0.9896849393844604, "metrics/particle/event/vector_pt_closure": 0.04935082793235779, "metrics/particle/matched/delta_eta_abs_mean": 0.006907064933329821, "metrics/particle/matched/delta_phi_abs_mean": 0.006783979944884777, "metrics/particle/matched/delta_r_mean": 0.010881814174354076, "metrics/particle/matched/energy_relative_abs_mean": 0.06186620891094208, "metrics/particle/matched/pid_accuracy": 0.9773755669593811, "metrics/particle/matched/pt_relative_abs_mean": 0.061918675899505615, "metrics/particle/matching/duplicate_fraction": 0.01983471028506756, "metrics/particle/matching/efficiency": 0.9193011522293091, "metrics/particle/matching/f1": 0.916252076625824, "metrics/particle/matching/purity": 0.913223147392273, "metrics/particle/matching/target_coverage_dr0p05": 0.90183025598526, "metrics/particle/matching/target_coverage_dr0p10": 0.9376040101051331, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8361064791679382, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9342762231826782}, "step_train_time": 0.8618199825286865, "step_valid_time": 1.8729981320066145, "step_total_time": 2087.98956823349, "elapsed_training_loop_seconds": 2087.98956823349, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0001350825626678, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.029310578862478676, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 440, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 432, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9775280898876404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9886363636363636, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9830508474576272, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.011363636363636364, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9707865168539326, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9818181818181818, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.976271186440678, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9775280898876404, "step_test_time": 3.1007313770096516, "step_plot_time": 3.1337180330010597, "step_validation_cycle_time": 8.685815308010206} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_27000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_27000.json new file mode 100644 index 0000000000000000000000000000000000000000..5a08f7ca667c7185dfb51f5e0afa61375ae76556 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_27000.json @@ -0,0 +1 @@ +{"epoch": 48.04270462633452, "train": {"Classification_binary": 0.01893242448568344, "Classification": 1.0612355936245876e-06, "Regression_pt": 0.011804903857409954, "Regression_eta": 6.68510256218724e-05, "Regression_sin_phi": 8.551574865123257e-05, "Regression_cos_phi": 3.242076127207838e-05, "Regression_energy": 0.01179599016904831, "Total": 0.41264915466308594, "Total_unweighted": 0.04271916728328051}, "train_group": {}, "valid": {"Classification_binary": 1.9486299753189087, "Classification": 0.026347514241933823, "Regression_pt": 0.09594228863716125, "Regression_eta": 0.0027515855617821217, "Regression_sin_phi": 0.0011734053259715438, "Regression_cos_phi": 0.0021135120186954737, "Regression_energy": 0.09511599689722061, "Total": 9.616022109985352, "Total_unweighted": 2.1720742780016735, "metrics/particle/class_1/efficiency": 0.9602510333061218, "metrics/particle/class_1/pid_accuracy": 0.9803921580314636, "metrics/particle/class_2/efficiency": 0.5357142686843872, "metrics/particle/class_2/pid_accuracy": 0.9333333373069763, "metrics/particle/class_3/efficiency": 0.8646034598350525, "metrics/particle/class_3/pid_accuracy": 0.9932885766029358, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.936170220375061, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9759036302566528, "metrics/particle/count/bias_mean": 0.01600000075995922, "metrics/particle/count/mae": 0.3440000116825104, "metrics/particle/count/prediction_mean": 4.823999881744385, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.051836464554071426, "metrics/particle/event/energy_response_mean": 0.9933437705039978, "metrics/particle/event/met_abs_error": 1.5310373306274414, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05192852020263672, "metrics/particle/event/scalar_pt_response_mean": 0.9919336438179016, "metrics/particle/event/vector_pt_closure": 0.05340419337153435, "metrics/particle/matched/delta_eta_abs_mean": 0.006347102578729391, "metrics/particle/matched/delta_phi_abs_mean": 0.0068814377300441265, "metrics/particle/matched/delta_r_mean": 0.010443051345646381, "metrics/particle/matched/energy_relative_abs_mean": 0.06126515567302704, "metrics/particle/matched/pid_accuracy": 0.9808743000030518, "metrics/particle/matched/pt_relative_abs_mean": 0.061250410974025726, "metrics/particle/matching/duplicate_fraction": 0.021558871492743492, "metrics/particle/matching/efficiency": 0.9134775400161743, "metrics/particle/matching/f1": 0.9119601249694824, "metrics/particle/matching/purity": 0.9104477763175964, "metrics/particle/matching/target_coverage_dr0p05": 0.9059900045394897, "metrics/particle/matching/target_coverage_dr0p10": 0.9351081252098083, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8311148285865784, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9309484362602234}, "step_train_time": 0.8208675384521484, "step_valid_time": 1.9483773810061393, "step_total_time": 2169.7826454639435, "elapsed_training_loop_seconds": 2169.7826454639435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9962001678667675, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02572441864778774, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 440, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9775280898876404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9886363636363636, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9830508474576272, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.011363636363636364, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9685393258426966, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9795454545454545, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9740112994350283, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9775280898876404, "step_test_time": 3.1952190779993543, "step_plot_time": 3.3591838749998715, "step_validation_cycle_time": 8.97859214899654} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_28000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_28000.json new file mode 100644 index 0000000000000000000000000000000000000000..a9020f3a485398894fb670a26b2859172db2379f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_28000.json @@ -0,0 +1 @@ +{"epoch": 49.8220640569395, "train": {"Classification_binary": 0.2450908124446869, "Classification": 1.2145064829383045e-05, "Regression_pt": 0.017357097938656807, "Regression_eta": 9.37014992814511e-05, "Regression_sin_phi": 7.392709812847897e-05, "Regression_cos_phi": 0.00012576249719131738, "Regression_energy": 0.017719795927405357, "Total": 0.8054771423339844, "Total_unweighted": 0.2804732424701797}, "train_group": {}, "valid": {"Classification_binary": 1.9183590412139893, "Classification": 0.02485993690788746, "Regression_pt": 0.09812181442975998, "Regression_eta": 0.002817378379404545, "Regression_sin_phi": 0.0010405945358797908, "Regression_cos_phi": 0.0018303956603631377, "Regression_energy": 0.09838360548019409, "Total": 9.30765151977539, "Total_unweighted": 2.1454127666074783, "metrics/particle/class_1/efficiency": 0.9602510333061218, "metrics/particle/class_1/pid_accuracy": 0.9825708270072937, "metrics/particle/class_2/efficiency": 0.5357142686843872, "metrics/particle/class_2/pid_accuracy": 0.9333333373069763, "metrics/particle/class_3/efficiency": 0.8607349991798401, "metrics/particle/class_3/pid_accuracy": 0.9932584166526794, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.9255319237709045, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9879518151283264, "metrics/particle/count/bias_mean": 0.024000000208616257, "metrics/particle/count/mae": 0.335999995470047, "metrics/particle/count/prediction_mean": 4.831999778747559, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.053028516471385956, "metrics/particle/event/energy_response_mean": 0.985409140586853, "metrics/particle/event/met_abs_error": 1.5351378917694092, "metrics/particle/event/scalar_pt_relative_abs_error": 0.052712809294462204, "metrics/particle/event/scalar_pt_response_mean": 0.9825501441955566, "metrics/particle/event/vector_pt_closure": 0.052190762013196945, "metrics/particle/matched/delta_eta_abs_mean": 0.006685339845716953, "metrics/particle/matched/delta_phi_abs_mean": 0.006961919832974672, "metrics/particle/matched/delta_r_mean": 0.010748964734375477, "metrics/particle/matched/energy_relative_abs_mean": 0.06406775861978531, "metrics/particle/matched/pid_accuracy": 0.9817517995834351, "metrics/particle/matched/pt_relative_abs_mean": 0.0641484186053276, "metrics/particle/matching/duplicate_fraction": 0.01986755058169365, "metrics/particle/matching/efficiency": 0.9118136167526245, "metrics/particle/matching/f1": 0.9095435738563538, "metrics/particle/matching/purity": 0.9072847962379456, "metrics/particle/matching/target_coverage_dr0p05": 0.90183025598526, "metrics/particle/matching/target_coverage_dr0p10": 0.9342762231826782, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8261231184005737, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9292845129966736}, "step_train_time": 0.8303279876708984, "step_valid_time": 1.9009883770049782, "step_total_time": 2250.100916624069, "elapsed_training_loop_seconds": 2250.100916624069, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9959565446967807, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.031920088014574244, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 439, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 432, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9775280898876404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9908883826879271, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9841628959276018, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.009111617312072893, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9707865168539326, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9840546697038725, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9773755656108597, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9775280898876404, "step_test_time": 3.091014389996417, "step_plot_time": 2.6878399230045034, "step_validation_cycle_time": 8.073824277991662} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_29000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_29000.json new file mode 100644 index 0000000000000000000000000000000000000000..295c4f69f62abd83599e8c340e5c9abc847c43ff --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_29000.json @@ -0,0 +1 @@ +{"epoch": 51.60142348754449, "train": {"Classification_binary": 0.06863963603973389, "Classification": 0.00044434386654756963, "Regression_pt": 0.013543575070798397, "Regression_eta": 0.0002357951016165316, "Regression_sin_phi": 0.00011407306737964973, "Regression_cos_phi": 0.00010224141442449763, "Regression_energy": 0.01422793511301279, "Total": 0.6815594434738159, "Total_unweighted": 0.09730759967351332}, "train_group": {}, "valid": {"Classification_binary": 2.041379451751709, "Classification": 0.028007477521896362, "Regression_pt": 0.10268912464380264, "Regression_eta": 0.0027023775037378073, "Regression_sin_phi": 0.0010971470037475228, "Regression_cos_phi": 0.00195030658505857, "Regression_energy": 0.10325119644403458, "Total": 9.853558540344238, "Total_unweighted": 2.2810770814539865, "metrics/particle/class_1/efficiency": 0.9644351601600647, "metrics/particle/class_1/pid_accuracy": 0.9761388301849365, "metrics/particle/class_2/efficiency": 0.6071428656578064, "metrics/particle/class_2/pid_accuracy": 0.9411764740943909, "metrics/particle/class_3/efficiency": 0.8781431317329407, "metrics/particle/class_3/pid_accuracy": 0.9911894202232361, "metrics/particle/class_4/efficiency": 0.96875, "metrics/particle/class_4/pid_accuracy": 0.9569892287254333, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9879518151283264, "metrics/particle/count/bias_mean": 0.03200000151991844, "metrics/particle/count/mae": 0.335999995470047, "metrics/particle/count/prediction_mean": 4.840000152587891, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.051029499620199203, "metrics/particle/event/energy_response_mean": 0.9880241751670837, "metrics/particle/event/met_abs_error": 1.5082849264144897, "metrics/particle/event/scalar_pt_relative_abs_error": 0.050380680710077286, "metrics/particle/event/scalar_pt_response_mean": 0.9867010116577148, "metrics/particle/event/vector_pt_closure": 0.05063270404934883, "metrics/particle/matched/delta_eta_abs_mean": 0.0064980811439454556, "metrics/particle/matched/delta_phi_abs_mean": 0.007103597279638052, "metrics/particle/matched/delta_r_mean": 0.010883812792599201, "metrics/particle/matched/energy_relative_abs_mean": 0.06401477754116058, "metrics/particle/matched/pid_accuracy": 0.9810469150543213, "metrics/particle/matched/pt_relative_abs_mean": 0.06380065530538559, "metrics/particle/matching/duplicate_fraction": 0.01570247858762741, "metrics/particle/matching/efficiency": 0.921796977519989, "metrics/particle/matching/f1": 0.9187396168708801, "metrics/particle/matching/purity": 0.9157024621963501, "metrics/particle/matching/target_coverage_dr0p05": 0.9076539278030396, "metrics/particle/matching/target_coverage_dr0p10": 0.940099835395813, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8344426155090332, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9384359121322632}, "step_train_time": 0.8522393703460693, "step_valid_time": 1.8989894139958778, "step_total_time": 2329.9580047130585, "elapsed_training_loop_seconds": 2329.9580047130585, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9960297004361921, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.031069889598400624, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 438, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9797752808988764, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9954337899543378, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9875424688561721, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0045662100456621, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9752808988764045, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9908675799086758, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9830124575311439, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9797752808988764, "step_test_time": 3.3017592660035007, "step_plot_time": 3.265549509000266, "step_validation_cycle_time": 8.930481682007667} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_3000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_3000.json new file mode 100644 index 0000000000000000000000000000000000000000..817d9589e01d789aa7590862a018a580092a7a4a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_3000.json @@ -0,0 +1 @@ +{"epoch": 5.338078291814947, "train": {"Classification_binary": 0.9403111934661865, "Classification": 0.022483740001916885, "Regression_pt": 0.05671503767371178, "Regression_eta": 0.0014786754036322236, "Regression_sin_phi": 0.00031887480872683227, "Regression_cos_phi": 0.0004134504124522209, "Regression_energy": 0.05944501608610153, "Total": 5.326852798461914, "Total_unweighted": 1.081165987852728}, "train_group": {}, "valid": {"Classification_binary": 1.0053073167800903, "Classification": 0.007410143036395311, "Regression_pt": 0.09439801424741745, "Regression_eta": 0.0026520518586039543, "Regression_sin_phi": 0.0008797423797659576, "Regression_cos_phi": 0.0008915899088606238, "Regression_energy": 0.09199416637420654, "Total": 6.192368984222412, "Total_unweighted": 1.2035330245853402, "metrics/particle/class_1/efficiency": 0.9623430967330933, "metrics/particle/class_1/pid_accuracy": 0.9739130139350891, "metrics/particle/class_2/efficiency": 0.6071428656578064, "metrics/particle/class_2/pid_accuracy": 0.8235294222831726, "metrics/particle/class_3/efficiency": 0.8704062104225159, "metrics/particle/class_3/pid_accuracy": 0.9977777600288391, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.9468085169792175, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.06400000303983688, "metrics/particle/count/mae": 0.2720000147819519, "metrics/particle/count/prediction_mean": 4.74399995803833, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.0488860048353672, "metrics/particle/event/energy_response_mean": 0.9922729134559631, "metrics/particle/event/met_abs_error": 1.3349552154541016, "metrics/particle/event/scalar_pt_relative_abs_error": 0.047957777976989746, "metrics/particle/event/scalar_pt_response_mean": 0.9922939538955688, "metrics/particle/event/vector_pt_closure": 0.04856734722852707, "metrics/particle/matched/delta_eta_abs_mean": 0.007985365577042103, "metrics/particle/matched/delta_phi_abs_mean": 0.007044852711260319, "metrics/particle/matched/delta_r_mean": 0.011971686966717243, "metrics/particle/matched/energy_relative_abs_mean": 0.06115075945854187, "metrics/particle/matched/pid_accuracy": 0.98097825050354, "metrics/particle/matched/pt_relative_abs_mean": 0.06123935431241989, "metrics/particle/matching/duplicate_fraction": 0.012647555209696293, "metrics/particle/matching/efficiency": 0.9184691905975342, "metrics/particle/matching/f1": 0.9246231317520142, "metrics/particle/matching/purity": 0.9308600425720215, "metrics/particle/matching/target_coverage_dr0p05": 0.9093177914619446, "metrics/particle/matching/target_coverage_dr0p10": 0.9434276223182678, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8427620530128479, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9292845129966736}, "step_train_time": 0.8453645706176758, "step_valid_time": 1.8830663630069466, "step_total_time": 234.82343912124634, "elapsed_training_loop_seconds": 234.82343912124634, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0004806371302457, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.03292058587076396, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9775280898876404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9977064220183486, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9875141884222475, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0022935779816513763, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9775280898876404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9977064220183486, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9875141884222475, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9775280898876404, "step_test_time": 3.0257356660004007, "step_plot_time": 2.6736392759921728, "step_validation_cycle_time": 7.978923499991652} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..bbc6dffd702403600b14b04b8870e1b19d22ae6b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 0.2559167742729187, "Classification": 2.0922130715916865e-05, "Regression_pt": 0.01624736562371254, "Regression_eta": 0.00036271283170208335, "Regression_sin_phi": 0.00021737343922723085, "Regression_cos_phi": 0.0002432630572002381, "Regression_energy": 0.0155018400400877, "Total": 1.0952390432357788, "Total_unweighted": 0.2885102513955644}, "train_group": {}, "valid": {"Classification_binary": 2.0429372787475586, "Classification": 0.03141220659017563, "Regression_pt": 0.09997390955686569, "Regression_eta": 0.002744319150224328, "Regression_sin_phi": 0.00106032972689718, "Regression_cos_phi": 0.0017571791540831327, "Regression_energy": 0.1004626527428627, "Total": 9.99101448059082, "Total_unweighted": 2.2803478756686673, "metrics/particle/class_1/efficiency": 0.9581589698791504, "metrics/particle/class_1/pid_accuracy": 0.9759825468063354, "metrics/particle/class_2/efficiency": 0.6071428656578064, "metrics/particle/class_2/pid_accuracy": 0.8823529481887817, "metrics/particle/class_3/efficiency": 0.8723404407501221, "metrics/particle/class_3/pid_accuracy": 0.9933481216430664, "metrics/particle/class_4/efficiency": 0.96875, "metrics/particle/class_4/pid_accuracy": 0.9139785170555115, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9879518151283264, "metrics/particle/count/bias_mean": 0.03999999910593033, "metrics/particle/count/mae": 0.35199999809265137, "metrics/particle/count/prediction_mean": 4.8480000495910645, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.052297141402959824, "metrics/particle/event/energy_response_mean": 1.0031932592391968, "metrics/particle/event/met_abs_error": 1.5260508060455322, "metrics/particle/event/scalar_pt_relative_abs_error": 0.052789587527513504, "metrics/particle/event/scalar_pt_response_mean": 1.00175940990448, "metrics/particle/event/vector_pt_closure": 0.051599420607089996, "metrics/particle/matched/delta_eta_abs_mean": 0.007048636209219694, "metrics/particle/matched/delta_phi_abs_mean": 0.007113788742572069, "metrics/particle/matched/delta_r_mean": 0.011179368011653423, "metrics/particle/matched/energy_relative_abs_mean": 0.06272056698799133, "metrics/particle/matched/pid_accuracy": 0.9773139953613281, "metrics/particle/matched/pt_relative_abs_mean": 0.06281625479459763, "metrics/particle/matching/duplicate_fraction": 0.020627062767744064, "metrics/particle/matching/efficiency": 0.9168053269386292, "metrics/particle/matching/f1": 0.9130074381828308, "metrics/particle/matching/purity": 0.9092409014701843, "metrics/particle/matching/target_coverage_dr0p05": 0.900166392326355, "metrics/particle/matching/target_coverage_dr0p10": 0.940099835395813, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8336106538772583, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9334442615509033}, "step_train_time": 0.9819962978363037, "step_valid_time": 2.0653405640041456, "step_total_time": 2411.762090444565, "elapsed_training_loop_seconds": 2411.762090444565, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0053583612432728, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0359334181423443, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 442, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 437, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9820224719101124, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9886877828054299, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9853438556933484, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.011312217194570135, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9775280898876404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9841628959276018, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9808342728297632, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9820224719101124, "step_test_time": 3.128793980999035, "step_plot_time": 2.826430347005953, "step_validation_cycle_time": 8.607809389999602} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_31000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_31000.json new file mode 100644 index 0000000000000000000000000000000000000000..95020138bd5bba5d364ee5f8fb4dda61724862e4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_31000.json @@ -0,0 +1 @@ +{"epoch": 55.16014234875445, "train": {"Classification_binary": 0.037607789039611816, "Classification": 5.9775931731564924e-05, "Regression_pt": 0.016689088195562363, "Regression_eta": 0.0002902581181842834, "Regression_sin_phi": 0.00013734030653722584, "Regression_cos_phi": 9.776058868737891e-05, "Regression_energy": 0.017097968608140945, "Total": 0.7322061657905579, "Total_unweighted": 0.07197998078845558}, "train_group": {}, "valid": {"Classification_binary": 2.2004871368408203, "Classification": 0.02791185863316059, "Regression_pt": 0.1079038754105568, "Regression_eta": 0.003394458210095763, "Regression_sin_phi": 0.001245219842530787, "Regression_cos_phi": 0.0019298081751912832, "Regression_energy": 0.10750962048768997, "Total": 10.5750732421875, "Total_unweighted": 2.4503819776000455, "metrics/particle/class_1/efficiency": 0.9539749026298523, "metrics/particle/class_1/pid_accuracy": 0.9802631735801697, "metrics/particle/class_2/efficiency": 0.5, "metrics/particle/class_2/pid_accuracy": 0.9285714030265808, "metrics/particle/class_3/efficiency": 0.8665376901626587, "metrics/particle/class_3/pid_accuracy": 0.9933035969734192, "metrics/particle/class_4/efficiency": 0.96875, "metrics/particle/class_4/pid_accuracy": 0.9354838728904724, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9759036302566528, "metrics/particle/count/bias_mean": -0.035999998450279236, "metrics/particle/count/mae": 0.3319999873638153, "metrics/particle/count/prediction_mean": 4.771999835968018, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.05098273977637291, "metrics/particle/event/energy_response_mean": 0.9853185415267944, "metrics/particle/event/met_abs_error": 1.5017086267471313, "metrics/particle/event/scalar_pt_relative_abs_error": 0.050704747438430786, "metrics/particle/event/scalar_pt_response_mean": 0.9825326204299927, "metrics/particle/event/vector_pt_closure": 0.05011840909719467, "metrics/particle/matched/delta_eta_abs_mean": 0.007534243166446686, "metrics/particle/matched/delta_phi_abs_mean": 0.007378338370472193, "metrics/particle/matched/delta_r_mean": 0.011736903339624405, "metrics/particle/matched/energy_relative_abs_mean": 0.06228400766849518, "metrics/particle/matched/pid_accuracy": 0.9808043837547302, "metrics/particle/matched/pt_relative_abs_mean": 0.062459103763103485, "metrics/particle/matching/duplicate_fraction": 0.012573344632983208, "metrics/particle/matching/efficiency": 0.9101497530937195, "metrics/particle/matching/f1": 0.9135699272155762, "metrics/particle/matching/purity": 0.9170159101486206, "metrics/particle/matching/target_coverage_dr0p05": 0.8901830315589905, "metrics/particle/matching/target_coverage_dr0p10": 0.9317803382873535, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8286189436912537, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9284525513648987}, "step_train_time": 0.8249099254608154, "step_valid_time": 2.0962891969975317, "step_total_time": 2496.9234631061554, "elapsed_training_loop_seconds": 2496.9234631061554, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9986593981437593, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.03275517122874039, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 439, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9752808988764045, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9886104783599089, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9819004524886877, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.011389521640091117, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9685393258426966, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9817767653758542, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9751131221719457, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9752808988764045, "step_test_time": 3.0869176339911064, "step_plot_time": 2.7830444959981833, "step_validation_cycle_time": 8.41354405900347} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_32000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_32000.json new file mode 100644 index 0000000000000000000000000000000000000000..cc48d5dfe3375b1e7fcdd09d43752117936d5380 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_32000.json @@ -0,0 +1 @@ +{"epoch": 56.93950177935943, "train": {"Classification_binary": 0.04844878241419792, "Classification": 9.499482075625565e-06, "Regression_pt": 0.012425893917679787, "Regression_eta": 0.00014988359180279076, "Regression_sin_phi": 9.973657870432362e-05, "Regression_cos_phi": 0.00016689387848600745, "Regression_energy": 0.012179732322692871, "Total": 0.559661328792572, "Total_unweighted": 0.07348042218563933}, "train_group": {}, "valid": {"Classification_binary": 2.4868342876434326, "Classification": 0.03254629671573639, "Regression_pt": 0.11371313780546188, "Regression_eta": 0.002655157120898366, "Regression_sin_phi": 0.0012613770086318254, "Regression_cos_phi": 0.0018538421718403697, "Regression_energy": 0.11374373733997345, "Total": 11.030888557434082, "Total_unweighted": 2.752607835805975, "metrics/particle/class_1/efficiency": 0.9539749026298523, "metrics/particle/class_1/pid_accuracy": 0.9802631735801697, "metrics/particle/class_2/efficiency": 0.6071428656578064, "metrics/particle/class_2/pid_accuracy": 0.8823529481887817, "metrics/particle/class_3/efficiency": 0.8607349991798401, "metrics/particle/class_3/pid_accuracy": 0.9910112619400024, "metrics/particle/class_4/efficiency": 0.9479166865348816, "metrics/particle/class_4/pid_accuracy": 0.9670329689979553, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9879518151283264, "metrics/particle/count/bias_mean": 0.00800000037997961, "metrics/particle/count/mae": 0.335999995470047, "metrics/particle/count/prediction_mean": 4.815999984741211, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.0507080964744091, "metrics/particle/event/energy_response_mean": 0.9914268255233765, "metrics/particle/event/met_abs_error": 1.5553101301193237, "metrics/particle/event/scalar_pt_relative_abs_error": 0.050491541624069214, "metrics/particle/event/scalar_pt_response_mean": 0.9902043342590332, "metrics/particle/event/vector_pt_closure": 0.051740311086177826, "metrics/particle/matched/delta_eta_abs_mean": 0.006289992015808821, "metrics/particle/matched/delta_phi_abs_mean": 0.006726578343659639, "metrics/particle/matched/delta_r_mean": 0.010293623432517052, "metrics/particle/matched/energy_relative_abs_mean": 0.059900082647800446, "metrics/particle/matched/pid_accuracy": 0.982600748538971, "metrics/particle/matched/pt_relative_abs_mean": 0.06006448343396187, "metrics/particle/matching/duplicate_fraction": 0.01744186133146286, "metrics/particle/matching/efficiency": 0.9084858298301697, "metrics/particle/matching/f1": 0.9077306985855103, "metrics/particle/matching/purity": 0.9069767594337463, "metrics/particle/matching/target_coverage_dr0p05": 0.9026622176170349, "metrics/particle/matching/target_coverage_dr0p10": 0.9342762231826782, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8244592547416687, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9292845129966736}, "step_train_time": 0.8472931385040283, "step_valid_time": 1.8914292909903452, "step_total_time": 2576.240321159363, "elapsed_training_loop_seconds": 2576.240321159363, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9995248439430351, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.031068301548057176, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 439, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9797752808988764, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9931662870159453, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9864253393665159, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.00683371298405467, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9752808988764045, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9886104783599089, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9819004524886877, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9797752808988764, "step_test_time": 3.1411154579982394, "step_plot_time": 3.1635942040011287, "step_validation_cycle_time": 8.675071464997018} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_4000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_4000.json new file mode 100644 index 0000000000000000000000000000000000000000..16aae16266ad8d3aeb0e7851df3082d42cf14fcc --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_4000.json @@ -0,0 +1 @@ +{"epoch": 7.117437722419929, "train": {"Classification_binary": 0.8083112239837646, "Classification": 0.0015772560145705938, "Regression_pt": 0.038426537066698074, "Regression_eta": 0.00041151774348691106, "Regression_sin_phi": 8.264669304480776e-05, "Regression_cos_phi": 0.0002606878988444805, "Regression_energy": 0.03821413218975067, "Total": 2.1774308681488037, "Total_unweighted": 0.8872840015901602}, "train_group": {}, "valid": {"Classification_binary": 0.9955715537071228, "Classification": 0.007439664099365473, "Regression_pt": 0.0960913673043251, "Regression_eta": 0.002801631111651659, "Regression_sin_phi": 0.0009683812386356294, "Regression_cos_phi": 0.0011304083745926619, "Regression_energy": 0.09328078478574753, "Total": 6.470292568206787, "Total_unweighted": 1.1972837906214409, "metrics/particle/class_1/efficiency": 0.9686192274093628, "metrics/particle/class_1/pid_accuracy": 0.9784017205238342, "metrics/particle/class_2/efficiency": 0.6071428656578064, "metrics/particle/class_2/pid_accuracy": 0.9411764740943909, "metrics/particle/class_3/efficiency": 0.8742746710777283, "metrics/particle/class_3/pid_accuracy": 0.997787594795227, "metrics/particle/class_4/efficiency": 0.96875, "metrics/particle/class_4/pid_accuracy": 0.9569892287254333, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.0560000017285347, "metrics/particle/count/mae": 0.25600001215934753, "metrics/particle/count/prediction_mean": 4.751999855041504, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.046449679881334305, "metrics/particle/event/energy_response_mean": 0.9911054968833923, "metrics/particle/event/met_abs_error": 1.2474840879440308, "metrics/particle/event/scalar_pt_relative_abs_error": 0.046224020421504974, "metrics/particle/event/scalar_pt_response_mean": 0.9912600517272949, "metrics/particle/event/vector_pt_closure": 0.04585873708128929, "metrics/particle/matched/delta_eta_abs_mean": 0.007849625311791897, "metrics/particle/matched/delta_phi_abs_mean": 0.006931285839527845, "metrics/particle/matched/delta_r_mean": 0.011649435386061668, "metrics/particle/matched/energy_relative_abs_mean": 0.06098111718893051, "metrics/particle/matched/pid_accuracy": 0.9855595827102661, "metrics/particle/matched/pt_relative_abs_mean": 0.0610012523829937, "metrics/particle/matching/duplicate_fraction": 0.01262626238167286, "metrics/particle/matching/efficiency": 0.921796977519989, "metrics/particle/matching/f1": 0.9271966814994812, "metrics/particle/matching/purity": 0.932659924030304, "metrics/particle/matching/target_coverage_dr0p05": 0.9101497530937195, "metrics/particle/matching/target_coverage_dr0p10": 0.9442595839500427, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8485856652259827, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9334442615509033}, "step_train_time": 0.8132879734039307, "step_valid_time": 1.8509066049882676, "step_total_time": 314.5030689239502, "elapsed_training_loop_seconds": 314.5030689239502, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9982058693152464, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.03220367377034661, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 437, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9775280898876404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9954233409610984, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9863945578231292, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.004576659038901602, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9775280898876404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9954233409610984, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9863945578231292, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9775280898876404, "step_test_time": 2.979002196996589, "step_plot_time": 2.6374438180064317, "step_validation_cycle_time": 7.811096796998754} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_5000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_5000.json new file mode 100644 index 0000000000000000000000000000000000000000..7a02600b2a01327fce4b64a1d973d1330caec33b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_5000.json @@ -0,0 +1 @@ +{"epoch": 8.896797153024911, "train": {"Classification_binary": 1.7411510944366455, "Classification": 0.008398886770009995, "Regression_pt": 0.22493307292461395, "Regression_eta": 0.0008466193685308099, "Regression_sin_phi": 0.00022552658629138023, "Regression_cos_phi": 0.0006886638584546745, "Regression_energy": 0.18310680985450745, "Total": 7.920928955078125, "Total_unweighted": 2.1593506737990538}, "train_group": {}, "valid": {"Classification_binary": 0.9712625741958618, "Classification": 0.007636113092303276, "Regression_pt": 0.09196688234806061, "Regression_eta": 0.0027561404276639223, "Regression_sin_phi": 0.0010349660879001021, "Regression_cos_phi": 0.0011118229012936354, "Regression_energy": 0.08963415026664734, "Total": 6.413678169250488, "Total_unweighted": 1.1654026493197307, "metrics/particle/class_1/efficiency": 0.9686192274093628, "metrics/particle/class_1/pid_accuracy": 0.9805615544319153, "metrics/particle/class_2/efficiency": 0.6071428656578064, "metrics/particle/class_2/pid_accuracy": 0.9411764740943909, "metrics/particle/class_3/efficiency": 0.8684719800949097, "metrics/particle/class_3/pid_accuracy": 0.993318498134613, "metrics/particle/class_4/efficiency": 0.96875, "metrics/particle/class_4/pid_accuracy": 0.9462365508079529, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.05999999865889549, "metrics/particle/count/mae": 0.2680000066757202, "metrics/particle/count/prediction_mean": 4.748000144958496, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.04887183755636215, "metrics/particle/event/energy_response_mean": 0.9963825941085815, "metrics/particle/event/met_abs_error": 1.3354071378707886, "metrics/particle/event/scalar_pt_relative_abs_error": 0.048275988548994064, "metrics/particle/event/scalar_pt_response_mean": 0.9953123927116394, "metrics/particle/event/vector_pt_closure": 0.047629717737436295, "metrics/particle/matched/delta_eta_abs_mean": 0.007271008100360632, "metrics/particle/matched/delta_phi_abs_mean": 0.007387623190879822, "metrics/particle/matched/delta_r_mean": 0.011584000661969185, "metrics/particle/matched/energy_relative_abs_mean": 0.06168743968009949, "metrics/particle/matched/pid_accuracy": 0.9837104082107544, "metrics/particle/matched/pt_relative_abs_mean": 0.06163728982210159, "metrics/particle/matching/duplicate_fraction": 0.013479359447956085, "metrics/particle/matching/efficiency": 0.9193011522293091, "metrics/particle/matching/f1": 0.9250732660293579, "metrics/particle/matching/purity": 0.930918276309967, "metrics/particle/matching/target_coverage_dr0p05": 0.9101497530937195, "metrics/particle/matching/target_coverage_dr0p10": 0.9434276223182678, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.841930091381073, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9309484362602234}, "step_train_time": 0.825228214263916, "step_valid_time": 1.8796034310071263, "step_total_time": 393.8646333217621, "elapsed_training_loop_seconds": 393.8646333217621, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0006626252229818, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.03323617932027534, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 439, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9797752808988764, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9931662870159453, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9864253393665159, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.00683371298405467, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9775280898876404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9908883826879271, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9841628959276018, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9797752808988764, "step_test_time": 3.098319184995489, "step_plot_time": 2.743943395005772, "step_validation_cycle_time": 8.137836783993407} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_6000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_6000.json new file mode 100644 index 0000000000000000000000000000000000000000..626cea7e07e97e36f5b0c3ef93d50b6cb4613e6b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_6000.json @@ -0,0 +1 @@ +{"epoch": 10.676156583629894, "train": {"Classification_binary": 1.9755384922027588, "Classification": 0.0019535780884325504, "Regression_pt": 0.06783148646354675, "Regression_eta": 0.0015144594945013523, "Regression_sin_phi": 0.004882081411778927, "Regression_cos_phi": 0.0010752223897725344, "Regression_energy": 0.06865938007831573, "Total": 9.679805755615234, "Total_unweighted": 2.1214547001291066}, "train_group": {}, "valid": {"Classification_binary": 0.9794872403144836, "Classification": 0.007890435867011547, "Regression_pt": 0.09273140877485275, "Regression_eta": 0.002849194221198559, "Regression_sin_phi": 0.0011206874623894691, "Regression_cos_phi": 0.0013140452792868018, "Regression_energy": 0.09013476967811584, "Total": 6.665830612182617, "Total_unweighted": 1.1755277815973386, "metrics/particle/class_1/efficiency": 0.9602510333061218, "metrics/particle/class_1/pid_accuracy": 0.9803921580314636, "metrics/particle/class_2/efficiency": 0.5714285969734192, "metrics/particle/class_2/pid_accuracy": 0.8125, "metrics/particle/class_3/efficiency": 0.8723404407501221, "metrics/particle/class_3/pid_accuracy": 0.9977827072143555, "metrics/particle/class_4/efficiency": 0.96875, "metrics/particle/class_4/pid_accuracy": 0.9462365508079529, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.052000001072883606, "metrics/particle/count/mae": 0.25999999046325684, "metrics/particle/count/prediction_mean": 4.75600004196167, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.04834207892417908, "metrics/particle/event/energy_response_mean": 0.9896279573440552, "metrics/particle/event/met_abs_error": 1.303029179573059, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04776516929268837, "metrics/particle/event/scalar_pt_response_mean": 0.9897123575210571, "metrics/particle/event/vector_pt_closure": 0.04786274582147598, "metrics/particle/matched/delta_eta_abs_mean": 0.007530312053859234, "metrics/particle/matched/delta_phi_abs_mean": 0.007301345933228731, "metrics/particle/matched/delta_r_mean": 0.01167035661637783, "metrics/particle/matched/energy_relative_abs_mean": 0.058881837874650955, "metrics/particle/matched/pid_accuracy": 0.9836660623550415, "metrics/particle/matched/pt_relative_abs_mean": 0.059023115783929825, "metrics/particle/matching/duplicate_fraction": 0.016820857301354408, "metrics/particle/matching/efficiency": 0.9168053269386292, "metrics/particle/matching/f1": 0.9217900633811951, "metrics/particle/matching/purity": 0.9268292784690857, "metrics/particle/matching/target_coverage_dr0p05": 0.9109817147254944, "metrics/particle/matching/target_coverage_dr0p10": 0.9434276223182678, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8502495884895325, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9292845129966736}, "step_train_time": 0.8494272232055664, "step_valid_time": 1.8998184650117764, "step_total_time": 474.218759059906, "elapsed_training_loop_seconds": 474.218759059906, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.998407419936184, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.03168637861276957, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 438, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9797752808988764, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9954337899543378, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9875424688561721, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0045662100456621, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9775280898876404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9931506849315068, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9852774631936579, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9797752808988764, "step_test_time": 3.026585878993501, "step_plot_time": 2.7013920530007454, "step_validation_cycle_time": 8.06340666700271} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_7000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_7000.json new file mode 100644 index 0000000000000000000000000000000000000000..0bdc087f508735e01fee115292c2077af0b7e912 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_7000.json @@ -0,0 +1 @@ +{"epoch": 12.455516014234876, "train": {"Classification_binary": 0.23204325139522552, "Classification": 0.004264019895344973, "Regression_pt": 0.03694182261824608, "Regression_eta": 0.0003764421271625906, "Regression_sin_phi": 0.00020556831441354007, "Regression_cos_phi": 0.0001840462937252596, "Regression_energy": 0.03653992712497711, "Total": 1.8712183237075806, "Total_unweighted": 0.3105550777690951}, "train_group": {}, "valid": {"Classification_binary": 0.9740707278251648, "Classification": 0.007968515157699585, "Regression_pt": 0.09037686139345169, "Regression_eta": 0.0026406224351376295, "Regression_sin_phi": 0.0010747264605015516, "Regression_cos_phi": 0.0010721961734816432, "Regression_energy": 0.08896905183792114, "Total": 6.389589786529541, "Total_unweighted": 1.166172701283358, "metrics/particle/class_1/efficiency": 0.9623430967330933, "metrics/particle/class_1/pid_accuracy": 0.9826086759567261, "metrics/particle/class_2/efficiency": 0.6071428656578064, "metrics/particle/class_2/pid_accuracy": 0.8235294222831726, "metrics/particle/class_3/efficiency": 0.8800773620605469, "metrics/particle/class_3/pid_accuracy": 0.997802197933197, "metrics/particle/class_4/efficiency": 0.96875, "metrics/particle/class_4/pid_accuracy": 0.9462365508079529, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.06400000303983688, "metrics/particle/count/mae": 0.25600001215934753, "metrics/particle/count/prediction_mean": 4.74399995803833, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.04606664553284645, "metrics/particle/event/energy_response_mean": 0.9919969439506531, "metrics/particle/event/met_abs_error": 1.3061332702636719, "metrics/particle/event/scalar_pt_relative_abs_error": 0.045798636972904205, "metrics/particle/event/scalar_pt_response_mean": 0.9921756982803345, "metrics/particle/event/vector_pt_closure": 0.04619218409061432, "metrics/particle/matched/delta_eta_abs_mean": 0.007254438009113073, "metrics/particle/matched/delta_phi_abs_mean": 0.007124008145183325, "metrics/particle/matched/delta_r_mean": 0.011384882964193821, "metrics/particle/matched/energy_relative_abs_mean": 0.06009892374277115, "metrics/particle/matched/pid_accuracy": 0.9846570491790771, "metrics/particle/matched/pt_relative_abs_mean": 0.060285989195108414, "metrics/particle/matching/duplicate_fraction": 0.015177065506577492, "metrics/particle/matching/efficiency": 0.921796977519989, "metrics/particle/matching/f1": 0.927973210811615, "metrics/particle/matching/purity": 0.9342327117919922, "metrics/particle/matching/target_coverage_dr0p05": 0.9134775400161743, "metrics/particle/matching/target_coverage_dr0p10": 0.9459234476089478, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8444259762763977, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9309484362602234}, "step_train_time": 0.8471930027008057, "step_valid_time": 1.8916256020020228, "step_total_time": 554.5191838741302, "elapsed_training_loop_seconds": 554.5191838741302, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.999721059088732, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.03380590391382321, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 440, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 438, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 437, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9842696629213483, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9954545454545455, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9898305084745763, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.004545454545454545, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9820224719101124, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9931818181818182, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9875706214689266, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9842696629213483, "step_test_time": 3.2728249350038823, "step_plot_time": 3.130880566997803, "step_validation_cycle_time": 8.735871836004662} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_8000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_8000.json new file mode 100644 index 0000000000000000000000000000000000000000..6d8e94ec569a28f5c60aef050daf87d682b1788b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_8000.json @@ -0,0 +1 @@ +{"epoch": 14.234875444839858, "train": {"Classification_binary": 1.0996499061584473, "Classification": 0.019876820966601372, "Regression_pt": 0.0654669851064682, "Regression_eta": 0.00030943992896936834, "Regression_sin_phi": 0.0005527244065888226, "Regression_cos_phi": 0.005120151210576296, "Regression_energy": 0.06342098861932755, "Total": 6.926441192626953, "Total_unweighted": 1.2543970163969789}, "train_group": {}, "valid": {"Classification_binary": 0.9770177602767944, "Classification": 0.008306003175675869, "Regression_pt": 0.092054583132267, "Regression_eta": 0.0026755754370242357, "Regression_sin_phi": 0.0010313137900084257, "Regression_cos_phi": 0.0011671322863548994, "Regression_energy": 0.0904569998383522, "Total": 6.466668128967285, "Total_unweighted": 1.172709367936477, "metrics/particle/class_1/efficiency": 0.9665272235870361, "metrics/particle/class_1/pid_accuracy": 0.976190447807312, "metrics/particle/class_2/efficiency": 0.6071428656578064, "metrics/particle/class_2/pid_accuracy": 0.8823529481887817, "metrics/particle/class_3/efficiency": 0.8704062104225159, "metrics/particle/class_3/pid_accuracy": 0.995555579662323, "metrics/particle/class_4/efficiency": 0.96875, "metrics/particle/class_4/pid_accuracy": 0.9569892287254333, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.07599999755620956, "metrics/particle/count/mae": 0.25200000405311584, "metrics/particle/count/prediction_mean": 4.73199987411499, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.04654007405042648, "metrics/particle/event/energy_response_mean": 0.9932532906532288, "metrics/particle/event/met_abs_error": 1.3630894422531128, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0465618297457695, "metrics/particle/event/scalar_pt_response_mean": 0.992851197719574, "metrics/particle/event/vector_pt_closure": 0.04711978882551193, "metrics/particle/matched/delta_eta_abs_mean": 0.007571870926767588, "metrics/particle/matched/delta_phi_abs_mean": 0.007061307784169912, "metrics/particle/matched/delta_r_mean": 0.011493059806525707, "metrics/particle/matched/energy_relative_abs_mean": 0.058933503925800323, "metrics/particle/matched/pid_accuracy": 0.9828054308891296, "metrics/particle/matched/pt_relative_abs_mean": 0.058921393007040024, "metrics/particle/matching/duplicate_fraction": 0.014370244927704334, "metrics/particle/matching/efficiency": 0.9193011522293091, "metrics/particle/matching/f1": 0.9266247153282166, "metrics/particle/matching/purity": 0.9340659379959106, "metrics/particle/matching/target_coverage_dr0p05": 0.9118136167526245, "metrics/particle/matching/target_coverage_dr0p10": 0.9459234476089478, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8494176268577576, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9301164746284485}, "step_train_time": 0.853783369064331, "step_valid_time": 1.872093665995635, "step_total_time": 635.6655077934265, "elapsed_training_loop_seconds": 635.6655077934265, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9998192862327381, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.03046027472669599, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 442, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 439, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 439, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9865168539325843, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9932126696832579, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9898534385569335, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.006787330316742082, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9865168539325843, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9932126696832579, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9898534385569335, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9865168539325843, "step_test_time": 3.011293948002276, "step_plot_time": 2.843513711995911, "step_validation_cycle_time": 8.17453850399761} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_9000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_9000.json new file mode 100644 index 0000000000000000000000000000000000000000..4d19988db5696ef244bca971b147eb14f373438c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/history/step_9000.json @@ -0,0 +1 @@ +{"epoch": 16.01423487544484, "train": {"Classification_binary": 0.5825812220573425, "Classification": 0.00014979293337091804, "Regression_pt": 0.016433145850896835, "Regression_eta": 0.00024352748005185276, "Regression_sin_phi": 7.104119140421972e-05, "Regression_cos_phi": 0.00021285572438500822, "Regression_energy": 0.016243161633610725, "Total": 1.229140281677246, "Total_unweighted": 0.6159347468710621}, "train_group": {}, "valid": {"Classification_binary": 0.984770655632019, "Classification": 0.007809602189809084, "Regression_pt": 0.0901467576622963, "Regression_eta": 0.0026637385599315166, "Regression_sin_phi": 0.0010587337892502546, "Regression_cos_phi": 0.0011813696473836899, "Regression_energy": 0.08862268179655075, "Total": 6.417680740356445, "Total_unweighted": 1.1762535392772406, "metrics/particle/class_1/efficiency": 0.9707112908363342, "metrics/particle/class_1/pid_accuracy": 0.9806034564971924, "metrics/particle/class_2/efficiency": 0.6785714030265808, "metrics/particle/class_2/pid_accuracy": 0.8421052694320679, "metrics/particle/class_3/efficiency": 0.8781431317329407, "metrics/particle/class_3/pid_accuracy": 0.9933920502662659, "metrics/particle/class_4/efficiency": 0.96875, "metrics/particle/class_4/pid_accuracy": 0.9569892287254333, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9879518151283264, "metrics/particle/count/bias_mean": -0.04800000041723251, "metrics/particle/count/mae": 0.2720000147819519, "metrics/particle/count/prediction_mean": 4.760000228881836, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.047461070120334625, "metrics/particle/event/energy_response_mean": 0.9957223534584045, "metrics/particle/event/met_abs_error": 1.4000351428985596, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04740764573216438, "metrics/particle/event/scalar_pt_response_mean": 0.9945758581161499, "metrics/particle/event/vector_pt_closure": 0.0479511059820652, "metrics/particle/matched/delta_eta_abs_mean": 0.006921912543475628, "metrics/particle/matched/delta_phi_abs_mean": 0.007109103724360466, "metrics/particle/matched/delta_r_mean": 0.011122560128569603, "metrics/particle/matched/energy_relative_abs_mean": 0.06058083474636078, "metrics/particle/matched/pid_accuracy": 0.9820305705070496, "metrics/particle/matched/pt_relative_abs_mean": 0.06067349761724472, "metrics/particle/matching/duplicate_fraction": 0.01512605044990778, "metrics/particle/matching/efficiency": 0.9259567260742188, "metrics/particle/matching/f1": 0.9306020140647888, "metrics/particle/matching/purity": 0.9352940917015076, "metrics/particle/matching/target_coverage_dr0p05": 0.9109817147254944, "metrics/particle/matching/target_coverage_dr0p10": 0.9459234476089478, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8502495884895325, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9367720484733582}, "step_train_time": 0.8975954055786133, "step_valid_time": 1.8747316840017447, "step_total_time": 716.6281044483185, "elapsed_training_loop_seconds": 716.6281044483185, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0015653177001205, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.030662507572905007, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 438, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 433, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9775280898876404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9931506849315068, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9852774631936579, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.00684931506849315, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9730337078651685, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9885844748858448, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9807474518686297, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9775280898876404, "step_test_time": 3.2273581659974298, "step_plot_time": 3.1975194300030125, "step_validation_cycle_time": 8.749196485994617} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..307b25d0dd132a8b5f23e5c2b31e407a691549cc --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_3mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 30, "checkpoint_freq": 2000, "val_freq": 1000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", "relaxed_load": false, "load_weights_only": true, "reset_task_loss_weights": true, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_z_tautau_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..9e32ec8808bb6238052c373c94aaa9e3fcea54b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0df2978b256461ac276e837c5c44dfdb6e0b8d21a82054f93b6b42f68bb6a1e0 +size 4506 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..aef94c35eb9561d3f9eefd9f528a0cce824e9602 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..6e296a97af565a6fbc2efb0d408ebbd8ea392e9f Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..abdd7d94b0273d185c515ad17b8705afd07f4566 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..77ea72d35c15384cca4ce52657d1a950d297599e Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8ea49227a71452ade50a56116720d26c739748b5 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..44790acae3b893cab5f1abb348201be94df54c86 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3de05550706eb024003c1cbde78b0f1b5a7c772b Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..bb1b064ff5d5f89390634d159aa6e87883854a6b Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ffba6a125075b89e3ad78cb651fd93daf5d67fbe Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..5e1eae1d02cf52e0cfc4a2cadb9c7104a1663726 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2f289aece18c3be39ca591ed00cc7bb5910146eb Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..b1d92317c7c7de6ee98941fea1cc9c6e9f195215 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/plots_step_32000/cld_edm_z_tautau_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/runs/train/events.out.tfevents.1794730213.workergpu212.1541065.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/runs/train/events.out.tfevents.1794730213.workergpu212.1541065.0 new file mode 100644 index 0000000000000000000000000000000000000000..56f0e4e68740fe01ae1bc938bcfb3315ae1feda1 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/runs/train/events.out.tfevents.1794730213.workergpu212.1541065.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fbe6c9f0f1213914d86608772b2d0e5904d894d41581eb9ee9c345d953ff89e +size 1173226 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/runs/valid/events.out.tfevents.1794730213.workergpu212.1541065.1 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/runs/valid/events.out.tfevents.1794730213.workergpu212.1541065.1 new file mode 100644 index 0000000000000000000000000000000000000000..c85f14caf1d3b5f9dee82bf5aaab1094b44007b8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/runs/valid/events.out.tfevents.1794730213.workergpu212.1541065.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28fb4dcbf985e7a05248ef0da25655b9c5265cc65b9ce442c7da110bd13e07fc +size 14594897 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..de8a9fe7d0a81dcad9d0f91b7247d43a4ca45963 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/scenario-manifest.json @@ -0,0 +1,1280 @@ +{ + "scenario": { + "name": "pf_fm_finetuning", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all" + }, + "allowed_variant_differences": [ + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets", + "data_config" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_finetuning", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_ecal_3mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_3mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all", + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_3mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 30, + "checkpoint_freq": 2000, + "val_freq": 1000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "relaxed_load": false, + "load_weights_only": true, + "reset_task_loss_weights": true, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_z_tautau_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_3mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--load", + "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "1000", + "--checkpoint_freq", + "2000", + "--patience", + "30", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--load_weights_only", + "true", + "--reset_task_loss_weights", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.feature_bag", + "true", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.trainable", + "all", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..968ee07632fa4da762ac1263a1221a885402e45b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_3mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_z_tautau_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +load_weights_only: true +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 30 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: true +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_z_tautau_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 1000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/train.log b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/train.log new file mode 100644 index 0000000000000000000000000000000000000000..cee6d5faab32879c5f37e134f60d836190dc432f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/train.log @@ -0,0 +1,2563 @@ +rank0 [2026-11-15 03:10:08,572] INFO : Will use single-gpu: NVIDIA H100 +rank0 [2026-11-15 03:10:08,572] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-15 03:10:09,725] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 03:10:09,725] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 03:10:09,725] INFO : Instantiating model +rank0 [2026-11-15 03:10:09,763] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 03:10:09,763] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 03:10:09,764] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 03:10:09,764] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 03:10:09,764] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 03:10:09,764] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 03:10:09,764] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:10:09,766] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 03:10:09,766] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:10:09,768] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 03:10:09,768] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:10:09,770] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 03:10:09,770] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:10:09,772] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 03:10:09,772] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:10:09,774] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 03:10:09,774] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:10:09,776] INFO : Backbone initialization took 0.01s +rank0 [2026-11-15 03:10:09,776] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 03:10:09,776] INFO : Initializing output DNNs +rank0 [2026-11-15 03:10:09,792] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:10:09,795] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:10:09,800] INFO : Output DNNs initialization took 0.02s +rank0 [2026-11-15 03:10:09,800] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 03:10:09,800] INFO : jagged_attention enabled=True +rank0 [2026-11-15 03:10:09,800] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 03:10:09,801] INFO : task_queries enabled=True +rank0 [2026-11-15 03:10:09,801] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 03:10:09,801] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 03:10:09,801] INFO : classification_query parameters: 256 +rank0 [2026-11-15 03:10:09,801] INFO : regression_query parameters: 256 +rank0 [2026-11-15 03:10:09,801] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 03:10:09,801] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 03:10:09,801] INFO : output_mode=elementwise +rank0 [2026-11-15 03:10:09,801] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 03:10:09,801] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 03:10:09,801] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 03:10:09,801] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 03:10:09,801] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 03:10:09,801] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 03:10:09,801] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 03:10:09,801] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 03:10:09,802] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 03:10:09,802] INFO : MLPF __init__ done +rank0 [2026-11-15 03:10:09,802] INFO : Instantiated model +rank0 [2026-11-15 03:10:09,802] INFO : Moving model to device rank=0 +rank0 [2026-11-15 03:10:09,813] INFO : Moved model to device rank=0 +rank0 [2026-11-15 03:10:09,814] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 03:10:09,925] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 03:10:09,925] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 03:10:09,925] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 03:10:10,004] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 03:10:10,005] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 03:10:10,006] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 03:10:10,006] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 03:10:10,006] INFO : Total parameters: 3910930 +rank0 [2026-11-15 03:10:10,007] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 03:10:10,008] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509 +rank0 [2026-11-15 03:10:10,009] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509 +rank0 [2026-11-15 03:10:10,009] INFO : Getting dataloaders +rank0 [2026-11-15 03:10:10,009] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:10:12,276] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:10:12,350] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:10:12,402] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 03:10:12,463] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:10:12,508] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 03:10:12,577] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 03:10:12,613] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:10:12,669] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 03:10:12,718] INFO : train_dataset: cld_edm_z_tautau_pf, 898 +rank0 [2026-11-15 03:10:12,770] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:10:12,770] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 03:10:12,770] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 03:10:12,770] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-15 03:10:12,770] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:10:12,785] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,799] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,813] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,827] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,841] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,855] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,868] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,882] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,896] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,911] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,911] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 03:10:12,911] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 03:10:12,962] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:10:13,110] INFO : Got dataloaders +rank0 [2026-11-15 03:10:13,111] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-15 03:10:13,111] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-15 03:10:13,111] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 03:10:13,129] INFO : Creating train_iterator +rank0 [2026-11-15 03:10:13,130] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:10:13,171] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:10:13,364] INFO : Created train_iterator +rank0 [2026-11-15 03:10:21,908] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.5670582455396653, 'Classification': 0.018406680354382844, 'Regression_pt': 0.1378782992810011, 'Regression_eta': 0.0033446101847221144, 'Regression_sin_phi': 0.0015505722568195778, 'Regression_cos_phi': 0.0040240935087786055, 'Regression_energy': 0.14219819964841007}, weights={'Classification_binary': 1.0, 'Classification': 85.13529968261719, 'Regression_pt': 11.365517616271973, 'Regression_eta': 468.53240966796875, 'Regression_sin_phi': 1000.0, 'Regression_cos_phi': 389.4189453125, 'Regression_energy': 11.02023983001709} +rank0 [2026-11-15 03:10:54,281] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 03:10:54,281] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:11:25,579] INFO : Step 1000/500000 rank0 | Train Loss: 5.2682 | Train Loss Unweighted: 2.1068 | Train Loss Components: Classification: 0.0028 | Classification_binary: 1.9324 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0844 | Regression_eta: 0.0005 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.2354 | Classification_binary: 1.9324 | Regression_cos_phi: 0.1922 | Regression_energy: 0.9305 | Regression_eta: 0.2113 | Regression_pt: 0.9706 | Regression_sin_phi: 0.7958 | LR: 2.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8121s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0022 | max_backward: 0.0252 | max_optimizer_step: 0.0780 | max_tensorboard_logging: 0.3046 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2616.5 | uss_MB: 1406.4 | child_rss_MB: 23279.7 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:11:25,581] INFO : Running validation on rank0 +rank0 [2026-11-15 03:11:25,581] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:11:25,581] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:11:25,781] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 03:11:25,955] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:11:26,011] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:11:26,056] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:11:26,099] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:11:26,137] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:11:26,185] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:11:26,230] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:11:26,273] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:11:26,327] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:11:26,379] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:11:26,418] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:11:26,460] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:11:26,494] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:11:26,534] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:11:26,575] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:11:26,613] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:11:26,615] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:11:26,617] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:11:26,619] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:11:26,620] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:11:26,622] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:11:26,624] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:11:26,624] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:11:26,626] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:11:26,627] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:11:26,628] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:11:26,629] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:11:26,631] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:11:26,632] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:11:26,634] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:11:26,635] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:11:27,717] INFO : VALIDATION | Step=1000/500000 | Train Loss=5.2682 | Valid Loss=6.4280 | Train Loss Unweighted=2.1068 | Valid Loss Unweighted=1.2495 | Stale=0 | ETA=620.3m +rank0 [2026-11-15 03:11:27,717] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:11:27,859] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:11:27,877] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:11:27,930] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:11:28,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:11:28,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:11:29,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:11:29,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:11:29,394] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:11:29,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:11:29,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:11:29,845] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:11:29,994] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:11:30,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:11:30,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:11:30,448] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:11:30,599] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:11:30,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:11:30,900] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:11:31,038] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:11:31,109] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:11:31,117] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:11:31,476] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:11:32,390] INFO : Plotted number of elements +rank0 [2026-11-15 03:11:33,266] INFO : Plotted jets +rank0 [2026-11-15 03:11:34,435] INFO : Plotted jet ratio +rank0 [2026-11-15 03:11:43,468] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 03:11:43,468] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:12:22,878] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 03:12:22,878] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:12:45,471] INFO : Step 2000/500000 rank0 | Train Loss: 3.6655 | Train Loss Unweighted: 0.9498 | Train Loss Components: Classification: 0.0081 | Classification_binary: 0.7749 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0816 | Regression_eta: 0.0003 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.6921 | Classification_binary: 0.7749 | Regression_cos_phi: 0.0676 | Regression_energy: 0.8988 | Regression_eta: 0.1267 | Regression_pt: 0.9617 | Regression_sin_phi: 0.1436 | LR: 2.02e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7679s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0232 | max_loss: 0.0022 | max_backward: 0.0261 | max_optimizer_step: 0.0776 | max_tensorboard_logging: 0.2807 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2733.4 | uss_MB: 1537.7 | child_rss_MB: 23031.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:12:45,473] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 03:12:45,523] INFO : Running validation on rank0 +rank0 [2026-11-15 03:12:45,523] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:12:45,523] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:12:45,603] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 03:12:45,677] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:12:45,746] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:12:45,792] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:12:45,837] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:12:45,882] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:12:45,927] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:12:45,972] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:12:46,045] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:12:46,114] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:12:46,181] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:12:46,227] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:12:46,271] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:12:46,315] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:12:46,360] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:12:46,405] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:12:46,475] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:12:46,479] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:12:46,481] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:12:46,483] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:12:46,484] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:12:46,486] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:12:46,487] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:12:46,488] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:12:46,489] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:12:46,490] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:12:46,491] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:12:46,492] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:12:46,494] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:12:46,495] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:12:46,497] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:12:46,498] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:12:47,426] INFO : VALIDATION | Step=2000/500000 | Train Loss=3.6655 | Valid Loss=6.1666 | Train Loss Unweighted=0.9498 | Valid Loss Unweighted=1.2067 | Stale=0 | ETA=640.3m +rank0 [2026-11-15 03:12:47,427] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:12:47,562] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:12:47,579] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:12:47,623] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:12:48,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:12:48,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:12:48,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:12:48,478] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:12:48,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:12:48,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:12:48,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:12:49,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:12:49,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:12:49,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:12:49,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:12:49,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:12:49,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:12:50,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:12:50,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:12:50,304] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:12:50,373] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:12:50,381] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:12:50,736] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:12:50,992] INFO : Plotted number of elements +rank0 [2026-11-15 03:12:51,832] INFO : Plotted jets +rank0 [2026-11-15 03:12:53,006] INFO : Plotted jet ratio +rank0 [2026-11-15 03:13:10,966] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 03:13:10,967] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:13:51,600] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 03:13:51,600] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:14:06,066] INFO : Step 3000/500000 rank0 | Train Loss: 5.3269 | Train Loss Unweighted: 1.0812 | Train Loss Components: Classification: 0.0225 | Classification_binary: 0.9403 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0594 | Regression_eta: 0.0015 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 1.9142 | Classification_binary: 0.9403 | Regression_cos_phi: 0.1610 | Regression_energy: 0.6551 | Regression_eta: 0.6928 | Regression_pt: 0.6446 | Regression_sin_phi: 0.3189 | LR: 2.05e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8446s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0022 | max_backward: 0.0254 | max_optimizer_step: 0.0800 | max_tensorboard_logging: 0.3206 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2786.2 | uss_MB: 1590.5 | child_rss_MB: 23032.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:14:06,068] INFO : Running validation on rank0 +rank0 [2026-11-15 03:14:06,068] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:14:06,068] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:14:06,149] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 03:14:06,225] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:14:06,295] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:14:06,344] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:14:06,390] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:14:06,435] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:14:06,480] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:14:06,526] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:14:06,601] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:14:06,670] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:14:06,740] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:14:06,787] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:14:06,833] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:14:06,879] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:14:06,925] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:14:06,971] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:14:07,042] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:14:07,046] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:14:07,048] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:14:07,050] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:14:07,051] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:14:07,052] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:14:07,054] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:14:07,055] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:14:07,056] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:14:07,057] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:14:07,058] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:14:07,059] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:14:07,061] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:14:07,062] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:14:07,064] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:14:07,065] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:14:07,958] INFO : VALIDATION | Step=3000/500000 | Train Loss=5.3269 | Valid Loss=6.1924 | Train Loss Unweighted=1.0812 | Valid Loss Unweighted=1.2035 | Stale=1 | ETA=648.4m +rank0 [2026-11-15 03:14:07,958] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:14:08,103] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:14:08,118] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:14:08,167] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:14:08,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:14:08,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:14:08,906] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:14:09,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:14:09,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:14:09,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:14:09,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:14:09,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:14:09,831] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:14:09,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:14:10,140] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:14:10,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:14:10,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:14:10,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:14:10,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:14:10,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:14:10,982] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:14:10,989] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:14:11,345] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:14:11,604] INFO : Plotted number of elements +rank0 [2026-11-15 03:14:12,447] INFO : Plotted jets +rank0 [2026-11-15 03:14:13,649] INFO : Plotted jet ratio +rank0 [2026-11-15 03:14:40,187] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 03:14:40,187] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:15:20,140] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 03:15:20,141] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:15:25,730] INFO : Step 4000/500000 rank0 | Train Loss: 2.1774 | Train Loss Unweighted: 0.8873 | Train Loss Components: Classification: 0.0016 | Classification_binary: 0.8083 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0382 | Regression_eta: 0.0004 | Regression_pt: 0.0384 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.1343 | Classification_binary: 0.8083 | Regression_cos_phi: 0.1015 | Regression_energy: 0.4211 | Regression_eta: 0.1928 | Regression_pt: 0.4367 | Regression_sin_phi: 0.0826 | LR: 2.08e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.8123s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0022 | max_backward: 0.0252 | max_optimizer_step: 0.0769 | max_tensorboard_logging: 0.3222 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2816.7 | uss_MB: 1621.1 | child_rss_MB: 23031.2 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:15:25,732] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 03:15:25,780] INFO : Running validation on rank0 +rank0 [2026-11-15 03:15:25,780] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:15:25,780] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:15:25,859] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 03:15:25,929] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:15:25,998] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:15:26,045] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:15:26,090] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:15:26,134] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:15:26,180] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:15:26,224] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:15:26,298] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:15:26,366] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:15:26,433] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:15:26,479] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:15:26,524] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:15:26,568] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:15:26,613] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:15:26,658] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:15:26,729] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:15:26,733] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:15:26,734] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:15:26,736] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:15:26,737] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:15:26,739] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:15:26,740] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:15:26,741] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:15:26,742] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:15:26,744] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:15:26,744] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:15:26,745] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:15:26,747] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:15:26,748] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:15:26,750] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:15:26,751] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:15:27,638] INFO : VALIDATION | Step=4000/500000 | Train Loss=2.1774 | Valid Loss=6.4703 | Train Loss Unweighted=0.8873 | Valid Loss Unweighted=1.1973 | Stale=2 | ETA=650.0m +rank0 [2026-11-15 03:15:27,638] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:15:27,783] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:15:27,799] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:15:27,852] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:15:28,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:15:28,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:15:28,558] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:15:28,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:15:28,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:15:29,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:15:29,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:15:29,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:15:29,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:15:29,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:15:29,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:15:29,923] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:15:30,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:15:30,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:15:30,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:15:30,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:15:30,615] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:15:30,622] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:15:30,977] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:15:31,240] INFO : Plotted number of elements +rank0 [2026-11-15 03:15:32,114] INFO : Plotted jets +rank0 [2026-11-15 03:15:33,247] INFO : Plotted jet ratio +rank0 [2026-11-15 03:16:08,648] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 03:16:08,649] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:16:45,111] INFO : Step 5000/500000 rank0 | Train Loss: 7.9209 | Train Loss Unweighted: 2.1594 | Train Loss Components: Classification: 0.0084 | Classification_binary: 1.7412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1831 | Regression_eta: 0.0008 | Regression_pt: 0.2249 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.7150 | Classification_binary: 1.7412 | Regression_cos_phi: 0.2682 | Regression_energy: 2.0179 | Regression_eta: 0.3967 | Regression_pt: 2.5565 | Regression_sin_phi: 0.2255 | LR: 2.13e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8242s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0245 | max_loss: 0.0023 | max_backward: 0.0255 | max_optimizer_step: 0.0814 | max_tensorboard_logging: 0.2972 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2840.4 | uss_MB: 1644.8 | child_rss_MB: 23031.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:16:45,113] INFO : Running validation on rank0 +rank0 [2026-11-15 03:16:45,113] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:16:45,113] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:16:45,193] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 03:16:45,264] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:16:45,335] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:16:45,384] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:16:45,430] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:16:45,475] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:16:45,521] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:16:45,567] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:16:45,642] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:16:45,712] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:16:45,779] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:16:45,827] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:16:45,873] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:16:45,918] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:16:45,964] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:16:46,010] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:16:46,082] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:16:46,086] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:16:46,087] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:16:46,089] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:16:46,090] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:16:46,092] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:16:46,093] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:16:46,095] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:16:46,095] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:16:46,096] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:16:46,098] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:16:46,098] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:16:46,101] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:16:46,101] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:16:46,103] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:16:46,104] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:16:47,000] INFO : VALIDATION | Step=5000/500000 | Train Loss=7.9209 | Valid Loss=6.4137 | Train Loss Unweighted=2.1594 | Valid Loss Unweighted=1.1654 | Stale=3 | ETA=649.9m +rank0 [2026-11-15 03:16:47,000] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:16:47,132] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:16:47,150] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:16:47,197] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:16:47,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:16:47,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:16:47,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:16:48,105] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:16:48,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:16:48,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:16:48,570] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:16:48,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:16:48,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:16:49,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:16:49,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:16:49,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:16:49,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:16:49,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:16:49,830] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:16:49,971] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:16:50,096] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:16:50,118] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:16:50,475] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:16:50,741] INFO : Plotted number of elements +rank0 [2026-11-15 03:16:51,620] INFO : Plotted jets +rank0 [2026-11-15 03:16:52,836] INFO : Plotted jet ratio +rank0 [2026-11-15 03:16:57,319] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 03:16:57,319] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:17:37,532] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 03:17:37,533] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:18:05,396] INFO : Step 6000/500000 rank0 | Train Loss: 9.6798 | Train Loss Unweighted: 2.1215 | Train Loss Components: Classification: 0.0020 | Classification_binary: 1.9755 | Regression_cos_phi: 0.0011 | Regression_energy: 0.0687 | Regression_eta: 0.0015 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.1663 | Classification_binary: 1.9755 | Regression_cos_phi: 0.4187 | Regression_energy: 0.7566 | Regression_eta: 0.7096 | Regression_pt: 0.7709 | Regression_sin_phi: 4.8821 | LR: 2.19e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8482s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0024 | max_backward: 0.0280 | max_optimizer_step: 0.1018 | max_tensorboard_logging: 0.2721 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2859.8 | uss_MB: 1664.3 | child_rss_MB: 23030.0 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:18:05,398] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 03:18:05,447] INFO : Running validation on rank0 +rank0 [2026-11-15 03:18:05,447] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:18:05,447] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:18:05,527] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 03:18:05,601] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:18:05,669] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:18:05,716] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:18:05,762] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:18:05,808] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:18:05,853] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:18:05,901] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:18:05,980] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:18:06,053] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:18:06,125] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:18:06,175] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:18:06,221] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:18:06,266] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:18:06,313] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:18:06,359] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:18:06,431] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:18:06,435] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:18:06,437] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:18:06,439] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:18:06,440] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:18:06,442] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:18:06,443] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:18:06,444] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:18:06,445] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:18:06,447] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:18:06,447] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:18:06,448] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:18:06,450] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:18:06,452] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:18:06,453] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:18:06,455] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:18:07,352] INFO : VALIDATION | Step=6000/500000 | Train Loss=9.6798 | Valid Loss=6.6658 | Train Loss Unweighted=2.1215 | Valid Loss Unweighted=1.1755 | Stale=4 | ETA=650.7m +rank0 [2026-11-15 03:18:07,353] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:18:07,484] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:18:07,500] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:18:07,553] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:18:07,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:18:08,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:18:08,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:18:08,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:18:08,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:18:08,771] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:18:08,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:18:09,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:18:09,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:18:09,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:18:09,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:18:09,699] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:18:09,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:18:10,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:18:10,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:18:10,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:18:10,378] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:18:10,389] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:18:10,745] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:18:11,004] INFO : Plotted number of elements +rank0 [2026-11-15 03:18:11,867] INFO : Plotted jets +rank0 [2026-11-15 03:18:13,072] INFO : Plotted jet ratio +rank0 [2026-11-15 03:18:26,707] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 03:18:26,707] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:19:06,618] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 03:19:06,618] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:19:25,754] INFO : Step 7000/500000 rank0 | Train Loss: 1.8712 | Train Loss Unweighted: 0.3106 | Train Loss Components: Classification: 0.0043 | Classification_binary: 0.2320 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0365 | Regression_eta: 0.0004 | Regression_pt: 0.0369 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.3630 | Classification_binary: 0.2320 | Regression_cos_phi: 0.0717 | Regression_energy: 0.4027 | Regression_eta: 0.1764 | Regression_pt: 0.4199 | Regression_sin_phi: 0.2056 | LR: 2.26e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8459s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0025 | max_backward: 0.0268 | max_optimizer_step: 0.0829 | max_tensorboard_logging: 0.3124 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2885.4 | uss_MB: 1689.9 | child_rss_MB: 23028.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:19:25,755] INFO : Running validation on rank0 +rank0 [2026-11-15 03:19:25,755] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:19:25,755] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:19:25,837] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 03:19:25,908] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:19:25,978] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:19:26,028] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:19:26,074] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:19:26,120] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:19:26,167] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:19:26,214] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:19:26,289] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:19:26,361] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:19:26,430] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:19:26,477] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:19:26,523] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:19:26,569] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:19:26,615] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:19:26,661] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:19:26,734] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:19:26,738] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:19:26,740] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:19:26,741] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:19:26,743] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:19:26,744] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:19:26,746] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:19:26,746] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:19:26,748] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:19:26,749] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:19:26,750] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:19:26,751] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:19:26,753] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:19:26,754] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:19:26,756] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:19:26,757] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:19:27,654] INFO : VALIDATION | Step=7000/500000 | Train Loss=1.8712 | Valid Loss=6.3896 | Train Loss Unweighted=0.3106 | Valid Loss Unweighted=1.1662 | Stale=5 | ETA=650.9m +rank0 [2026-11-15 03:19:27,654] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:19:27,797] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:19:27,815] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:19:27,863] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:19:28,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:19:28,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:19:28,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:19:28,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:19:28,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:19:29,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:19:29,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:19:29,478] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:19:29,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:19:29,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:19:29,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:19:30,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:19:30,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:19:30,496] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:19:30,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:19:30,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:19:30,925] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:19:30,944] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:19:31,321] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:19:31,617] INFO : Plotted number of elements +rank0 [2026-11-15 03:19:32,636] INFO : Plotted jets +rank0 [2026-11-15 03:19:34,050] INFO : Plotted jet ratio +rank0 [2026-11-15 03:19:56,407] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 03:19:56,407] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:20:36,564] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 03:20:36,565] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:20:46,869] INFO : Step 8000/500000 rank0 | Train Loss: 6.9264 | Train Loss Unweighted: 1.2544 | Train Loss Components: Classification: 0.0199 | Classification_binary: 1.0996 | Regression_cos_phi: 0.0051 | Regression_energy: 0.0634 | Regression_eta: 0.0003 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 1.6922 | Classification_binary: 1.0996 | Regression_cos_phi: 1.9939 | Regression_energy: 0.6989 | Regression_eta: 0.1450 | Regression_pt: 0.7441 | Regression_sin_phi: 0.5527 | LR: 2.34e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8530s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0279 | max_loss: 0.0025 | max_backward: 0.0281 | max_optimizer_step: 0.0846 | max_tensorboard_logging: 0.3033 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2920.6 | uss_MB: 1725.1 | child_rss_MB: 23026.8 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:20:46,871] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 03:20:46,921] INFO : Running validation on rank0 +rank0 [2026-11-15 03:20:46,921] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:20:46,921] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:20:46,999] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 03:20:47,071] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:20:47,139] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:20:47,187] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:20:47,233] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:20:47,278] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:20:47,324] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:20:47,370] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:20:47,445] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:20:47,514] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:20:47,581] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:20:47,628] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:20:47,672] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:20:47,717] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:20:47,763] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:20:47,808] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:20:47,882] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:20:47,886] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:20:47,887] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:20:47,889] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:20:47,891] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:20:47,892] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:20:47,894] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:20:47,895] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:20:47,897] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:20:47,898] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:20:47,899] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:20:47,900] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:20:47,902] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:20:47,903] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:20:47,904] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:20:47,906] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:20:48,798] INFO : VALIDATION | Step=8000/500000 | Train Loss=6.9264 | Valid Loss=6.4667 | Train Loss Unweighted=1.2544 | Valid Loss Unweighted=1.1727 | Stale=6 | ETA=651.6m +rank0 [2026-11-15 03:20:48,798] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:20:48,936] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:20:48,951] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:48,998] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:20:49,419] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:20:49,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:20:49,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:20:49,881] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:20:50,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:20:50,186] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:20:50,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:20:50,492] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:20:50,645] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:20:50,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:20:50,952] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:20:51,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:20:51,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:20:51,408] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:20:51,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:20:51,699] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:20:51,807] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:20:51,814] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:52,176] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:52,439] INFO : Plotted number of elements +rank0 [2026-11-15 03:20:53,309] INFO : Plotted jets +rank0 [2026-11-15 03:20:54,646] INFO : Plotted jet ratio +rank0 [2026-11-15 03:21:26,011] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 03:21:26,011] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:22:06,466] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 03:22:06,466] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:22:07,880] INFO : Step 9000/500000 rank0 | Train Loss: 1.2291 | Train Loss Unweighted: 0.6159 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.5826 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0162 | Regression_eta: 0.0002 | Regression_pt: 0.0164 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0128 | Classification_binary: 0.5826 | Regression_cos_phi: 0.0829 | Regression_energy: 0.1790 | Regression_eta: 0.1141 | Regression_pt: 0.1868 | Regression_sin_phi: 0.0710 | LR: 2.43e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.8971s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0270 | max_optimizer_step: 0.0873 | max_tensorboard_logging: 0.3584 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2868.3 | uss_MB: 1672.8 | child_rss_MB: 23028.8 | allocated_MB: 170.0 | reserved_MB: 296.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:22:07,881] INFO : Running validation on rank0 +rank0 [2026-11-15 03:22:07,881] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:22:07,881] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:22:07,963] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 03:22:08,035] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:22:08,104] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:22:08,153] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:22:08,199] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:22:08,244] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:22:08,290] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:22:08,337] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:22:08,414] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:22:08,484] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:22:08,551] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:22:08,598] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:22:08,644] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:22:08,689] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:22:08,735] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:22:08,780] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:22:08,844] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:22:08,848] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:22:08,849] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:22:08,851] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:22:08,853] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:22:08,854] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:22:08,856] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:22:08,857] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:22:08,858] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:22:08,860] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:22:08,860] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:22:08,862] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:22:08,863] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:22:08,865] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:22:08,866] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:22:08,868] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:22:09,760] INFO : VALIDATION | Step=9000/500000 | Train Loss=1.2291 | Valid Loss=6.4177 | Train Loss Unweighted=0.6159 | Valid Loss Unweighted=1.1763 | Stale=7 | ETA=651.6m +rank0 [2026-11-15 03:22:09,761] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:22:09,920] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:22:09,935] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:09,984] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:22:10,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:22:10,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:22:10,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:22:10,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:22:11,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:22:11,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:22:11,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:22:11,595] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:22:11,759] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:22:11,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:22:12,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:22:12,258] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:22:12,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:22:12,578] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:22:12,737] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:22:12,906] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:22:12,986] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:22:12,994] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:13,392] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:13,682] INFO : Plotted number of elements +rank0 [2026-11-15 03:22:14,739] INFO : Plotted jets +rank0 [2026-11-15 03:22:16,178] INFO : Plotted jet ratio +rank0 [2026-11-15 03:22:55,212] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 03:22:55,212] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:23:27,412] INFO : Step 10000/500000 rank0 | Train Loss: 3.2697 | Train Loss Unweighted: 0.5968 | Train Loss Components: Classification: 0.0133 | Classification_binary: 0.5215 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0311 | Regression_eta: 0.0008 | Regression_pt: 0.0292 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 1.1292 | Classification_binary: 0.5215 | Regression_cos_phi: 0.2202 | Regression_energy: 0.3428 | Regression_eta: 0.3871 | Regression_pt: 0.3317 | Regression_sin_phi: 0.3373 | LR: 2.52e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8388s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0023 | max_backward: 0.0257 | max_optimizer_step: 0.0826 | max_tensorboard_logging: 0.3124 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2965.7 | uss_MB: 1770.2 | child_rss_MB: 23034.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:23:27,414] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 03:23:27,465] INFO : Running validation on rank0 +rank0 [2026-11-15 03:23:27,465] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:23:27,465] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:23:27,544] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 03:23:27,616] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:23:27,685] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:23:27,734] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:23:27,780] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:23:27,825] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:23:27,871] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:23:27,920] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:23:27,996] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:23:28,067] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:23:28,137] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:23:28,185] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:23:28,230] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:23:28,275] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:23:28,321] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:23:28,367] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:23:28,442] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:23:28,446] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:23:28,448] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:23:28,449] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:23:28,451] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:23:28,452] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:23:28,454] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:23:28,454] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:23:28,456] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:23:28,457] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:23:28,458] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:23:28,459] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:23:28,461] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:23:28,462] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:23:28,464] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:23:28,465] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:23:29,368] INFO : VALIDATION | Step=10000/500000 | Train Loss=3.2697 | Valid Loss=6.5776 | Train Loss Unweighted=0.5968 | Valid Loss Unweighted=1.1829 | Stale=8 | ETA=650.3m +rank0 [2026-11-15 03:23:29,368] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:23:29,512] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:23:29,527] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:23:29,578] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:23:30,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:23:30,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:23:30,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:23:30,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:23:30,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:23:30,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:23:30,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:23:31,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:23:31,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:23:31,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:23:31,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:23:31,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:23:31,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:23:31,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:23:32,137] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:23:32,275] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:23:32,386] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:23:32,393] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:23:32,743] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:23:33,003] INFO : Plotted number of elements +rank0 [2026-11-15 03:23:33,850] INFO : Plotted jets +rank0 [2026-11-15 03:23:35,079] INFO : Plotted jet ratio +rank0 [2026-11-15 03:23:43,674] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 03:23:43,674] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:24:24,149] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 03:24:24,149] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:24:48,037] INFO : Step 11000/500000 rank0 | Train Loss: 2.7025 | Train Loss Unweighted: 0.5553 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.4838 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0343 | Regression_eta: 0.0002 | Regression_pt: 0.0352 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0158 | Classification_binary: 0.4838 | Regression_cos_phi: 0.2055 | Regression_energy: 0.3785 | Regression_eta: 0.0923 | Regression_pt: 0.3999 | Regression_sin_phi: 1.1266 | LR: 2.63e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8432s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0838 | max_tensorboard_logging: 0.3208 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3000.8 | uss_MB: 1805.2 | child_rss_MB: 23033.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:24:48,038] INFO : Running validation on rank0 +rank0 [2026-11-15 03:24:48,038] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:24:48,038] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:24:48,118] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 03:24:48,190] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:24:48,260] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:24:48,310] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:24:48,355] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:24:48,400] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:24:48,446] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:24:48,493] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:24:48,568] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:24:48,637] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:24:48,705] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:24:48,752] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:24:48,798] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:24:48,843] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:24:48,889] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:24:48,934] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:24:49,008] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:24:49,012] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:24:49,014] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:24:49,016] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:24:49,017] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:24:49,018] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:24:49,020] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:24:49,020] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:24:49,022] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:24:49,023] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:24:49,024] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:24:49,025] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:24:49,027] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:24:49,028] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:24:49,029] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:24:49,031] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:24:49,915] INFO : VALIDATION | Step=11000/500000 | Train Loss=2.7025 | Valid Loss=6.7501 | Train Loss Unweighted=0.5553 | Valid Loss Unweighted=1.1630 | Stale=9 | ETA=649.6m +rank0 [2026-11-15 03:24:49,916] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:24:50,063] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:24:50,091] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:50,138] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:24:50,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:24:50,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:24:50,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:24:51,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:24:51,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:24:51,330] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:24:51,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:24:51,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:24:51,785] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:24:51,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:24:52,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:24:52,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:24:52,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:24:52,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:24:52,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:24:52,830] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:24:52,891] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:24:52,898] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:53,253] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:53,512] INFO : Plotted number of elements +rank0 [2026-11-15 03:24:54,356] INFO : Plotted jets +rank0 [2026-11-15 03:24:55,499] INFO : Plotted jet ratio +rank0 [2026-11-15 03:25:12,817] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 03:25:12,817] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:25:52,744] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 03:25:52,745] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:26:07,677] INFO : Step 12000/500000 rank0 | Train Loss: 1.5575 | Train Loss Unweighted: 0.7127 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.6898 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0102 | Regression_eta: 0.0003 | Regression_pt: 0.0117 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0040 | Classification_binary: 0.6898 | Regression_cos_phi: 0.1119 | Regression_energy: 0.1123 | Regression_eta: 0.1440 | Regression_pt: 0.1330 | Regression_sin_phi: 0.3627 | LR: 2.75e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7700s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0269 | max_optimizer_step: 0.0831 | max_tensorboard_logging: 0.2250 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3012.3 | uss_MB: 1816.8 | child_rss_MB: 23033.4 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:26:07,679] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 03:26:07,729] INFO : Running validation on rank0 +rank0 [2026-11-15 03:26:07,729] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:26:07,729] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:26:07,808] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 03:26:07,883] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:26:07,953] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:26:08,003] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:26:08,050] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:26:08,096] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:26:08,142] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:26:08,189] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:26:08,263] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:26:08,333] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:26:08,402] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:26:08,450] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:26:08,495] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:26:08,541] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:26:08,587] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:26:08,633] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:26:08,710] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:26:08,714] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:26:08,716] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:26:08,717] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:26:08,719] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:26:08,720] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:26:08,722] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:26:08,722] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:26:08,724] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:26:08,725] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:26:08,726] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:26:08,727] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:26:08,729] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:26:08,730] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:26:08,732] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:26:08,733] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:26:09,632] INFO : VALIDATION | Step=12000/500000 | Train Loss=1.5575 | Valid Loss=7.1893 | Train Loss Unweighted=0.7127 | Valid Loss Unweighted=1.1955 | Stale=10 | ETA=648.3m +rank0 [2026-11-15 03:26:09,632] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:26:09,736] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:26:09,752] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:09,802] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:26:10,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:26:10,408] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:26:10,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:26:10,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:26:10,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:26:11,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:26:11,173] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:26:11,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:26:11,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:26:11,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:26:11,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:26:11,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:26:12,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:26:12,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:26:12,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:26:12,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:26:12,646] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:26:12,653] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:13,004] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:13,265] INFO : Plotted number of elements +rank0 [2026-11-15 03:26:14,114] INFO : Plotted jets +rank0 [2026-11-15 03:26:15,341] INFO : Plotted jet ratio +rank0 [2026-11-15 03:26:41,987] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 03:26:41,988] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:27:22,339] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 03:27:22,339] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:27:28,462] INFO : Step 13000/500000 rank0 | Train Loss: 2.3960 | Train Loss Unweighted: 1.4400 | Train Loss Components: Classification: 0.0007 | Classification_binary: 1.3922 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0229 | Regression_eta: 0.0003 | Regression_pt: 0.0234 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 1.3922 | Regression_cos_phi: 0.1759 | Regression_energy: 0.2524 | Regression_eta: 0.1259 | Regression_pt: 0.2655 | Regression_sin_phi: 0.1252 | LR: 2.88e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7432s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0022 | max_backward: 0.0264 | max_optimizer_step: 0.0819 | max_tensorboard_logging: 0.2271 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3047.3 | uss_MB: 1851.8 | child_rss_MB: 23032.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:27:28,463] INFO : Running validation on rank0 +rank0 [2026-11-15 03:27:28,463] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:27:28,463] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:27:28,541] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 03:27:28,611] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:27:28,679] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:27:28,728] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:27:28,774] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:27:28,819] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:27:28,864] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:27:28,910] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:27:28,984] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:27:29,053] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:27:29,121] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:27:29,167] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:27:29,211] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:27:29,256] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:27:29,302] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:27:29,347] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:27:29,422] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:27:29,426] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:27:29,428] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:27:29,430] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:27:29,431] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:27:29,433] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:27:29,434] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:27:29,435] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:27:29,436] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:27:29,437] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:27:29,438] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:27:29,440] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:27:29,441] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:27:29,443] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:27:29,444] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:27:29,446] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:27:30,334] INFO : VALIDATION | Step=13000/500000 | Train Loss=2.3960 | Valid Loss=6.9363 | Train Loss Unweighted=1.4400 | Valid Loss Unweighted=1.2287 | Stale=11 | ETA=647.6m +rank0 [2026-11-15 03:27:30,334] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:27:30,440] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:27:30,455] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:27:30,500] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:27:30,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:27:31,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:27:31,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:27:31,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:27:31,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:27:31,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:27:31,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:27:31,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:27:32,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:27:32,276] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:27:32,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:27:32,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:27:32,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:27:32,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:27:33,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:27:33,173] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:27:33,239] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:27:33,246] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:27:33,607] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:27:33,868] INFO : Plotted number of elements +rank0 [2026-11-15 03:27:34,707] INFO : Plotted jets +rank0 [2026-11-15 03:27:35,843] INFO : Plotted jet ratio +rank0 [2026-11-15 03:28:11,265] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 03:28:11,265] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:28:48,883] INFO : Step 14000/500000 rank0 | Train Loss: 2.6940 | Train Loss Unweighted: 0.5470 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.3921 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0766 | Regression_eta: 0.0002 | Regression_pt: 0.0773 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0103 | Classification_binary: 0.3921 | Regression_cos_phi: 0.1141 | Regression_energy: 0.8443 | Regression_eta: 0.0928 | Regression_pt: 0.8780 | Regression_sin_phi: 0.3624 | LR: 3.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7707s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0022 | max_backward: 0.0268 | max_optimizer_step: 0.0855 | max_tensorboard_logging: 0.2315 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3066.5 | uss_MB: 1870.9 | child_rss_MB: 23031.7 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:28:48,891] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 03:28:48,940] INFO : Running validation on rank0 +rank0 [2026-11-15 03:28:48,940] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:28:48,940] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:28:49,022] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 03:28:49,097] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:28:49,166] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:28:49,216] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:28:49,262] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:28:49,308] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:28:49,354] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:28:49,401] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:28:49,477] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:28:49,553] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:28:49,623] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:28:49,669] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:28:49,715] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:28:49,760] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:28:49,807] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:28:49,853] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:28:49,930] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:28:49,935] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:28:49,936] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:28:49,938] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:28:49,939] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:28:49,941] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:28:49,942] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:28:49,944] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:28:49,945] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:28:49,947] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:28:49,947] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:28:49,949] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:28:49,950] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:28:49,952] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:28:49,953] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:28:49,955] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:28:50,850] INFO : VALIDATION | Step=14000/500000 | Train Loss=2.6940 | Valid Loss=7.2250 | Train Loss Unweighted=0.5470 | Valid Loss Unweighted=1.2046 | Stale=12 | ETA=646.7m +rank0 [2026-11-15 03:28:50,851] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:28:50,953] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:28:50,968] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:51,018] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:28:51,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:28:51,614] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:28:51,768] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:28:51,923] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:28:52,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:28:52,229] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:28:52,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:28:52,544] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:28:52,700] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:28:52,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:28:53,009] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:28:53,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:28:53,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:28:53,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:28:53,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:28:53,764] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:28:53,865] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:28:53,877] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:54,238] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:54,502] INFO : Plotted number of elements +rank0 [2026-11-15 03:28:55,363] INFO : Plotted jets +rank0 [2026-11-15 03:28:56,576] INFO : Plotted jet ratio +rank0 [2026-11-15 03:29:00,606] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 03:29:00,606] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:29:41,061] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 03:29:41,061] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:30:09,541] INFO : Step 15000/500000 rank0 | Train Loss: 1.7450 | Train Loss Unweighted: 0.8081 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.7603 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0232 | Regression_eta: 0.0001 | Regression_pt: 0.0237 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0023 | Classification_binary: 0.7603 | Regression_cos_phi: 0.2117 | Regression_energy: 0.2555 | Regression_eta: 0.0594 | Regression_pt: 0.2696 | Regression_sin_phi: 0.1862 | LR: 3.17e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7512s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0022 | max_backward: 0.0266 | max_optimizer_step: 0.0851 | max_tensorboard_logging: 0.2299 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3096.5 | uss_MB: 1901.0 | child_rss_MB: 23030.7 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:30:09,542] INFO : Running validation on rank0 +rank0 [2026-11-15 03:30:09,543] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:30:09,543] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:30:09,625] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 03:30:09,697] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:30:09,767] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:30:09,818] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:30:09,864] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:30:09,909] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:30:09,956] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:30:10,003] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:30:10,080] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:30:10,151] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:30:10,220] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:30:10,266] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:30:10,312] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:30:10,357] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:30:10,402] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:30:10,447] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:30:10,510] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:30:10,514] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:30:10,516] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:30:10,517] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:30:10,519] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:30:10,520] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:30:10,522] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:30:10,522] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:30:10,524] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:30:10,525] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:30:10,526] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:30:10,527] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:30:10,529] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:30:10,529] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:30:10,530] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:30:10,532] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:30:11,425] INFO : VALIDATION | Step=15000/500000 | Train Loss=1.7450 | Valid Loss=7.3469 | Train Loss Unweighted=0.8081 | Valid Loss Unweighted=1.2934 | Stale=13 | ETA=645.7m +rank0 [2026-11-15 03:30:11,426] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:30:11,531] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:30:11,551] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:11,596] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:30:12,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:30:12,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:30:12,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:30:12,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:30:12,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:30:12,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:30:12,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:30:13,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:30:13,245] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:30:13,399] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:30:13,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:30:13,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:30:13,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:30:14,011] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:30:14,164] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:30:14,303] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:30:14,372] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:30:14,382] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:14,745] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:15,006] INFO : Plotted number of elements +rank0 [2026-11-15 03:30:15,853] INFO : Plotted jets +rank0 [2026-11-15 03:30:17,011] INFO : Plotted jet ratio +rank0 [2026-11-15 03:30:29,505] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 03:30:29,505] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:31:10,566] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 03:31:10,566] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:31:30,404] INFO : Step 16000/500000 rank0 | Train Loss: 3.9820 | Train Loss Unweighted: 0.9496 | Train Loss Components: Classification: 0.0007 | Classification_binary: 0.7415 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1020 | Regression_eta: 0.0007 | Regression_pt: 0.1036 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.7415 | Regression_cos_phi: 0.3641 | Regression_energy: 1.1236 | Regression_eta: 0.3266 | Regression_pt: 1.1771 | Regression_sin_phi: 0.1891 | LR: 3.33e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.8017s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0023 | max_backward: 0.0257 | max_optimizer_step: 0.0829 | max_tensorboard_logging: 0.2689 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3121.3 | uss_MB: 1925.7 | child_rss_MB: 23028.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:31:30,412] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 03:31:30,462] INFO : Running validation on rank0 +rank0 [2026-11-15 03:31:30,462] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:31:30,462] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:31:30,544] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 03:31:30,616] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:31:30,686] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:31:30,735] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:31:30,782] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:31:30,827] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:31:30,873] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:31:30,919] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:31:30,994] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:31:31,065] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:31:31,133] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:31:31,180] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:31:31,225] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:31:31,270] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:31:31,316] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:31:31,361] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:31:31,426] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:31:31,430] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:31:31,431] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:31:31,433] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:31:31,434] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:31:31,436] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:31:31,437] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:31:31,438] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:31:31,439] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:31:31,441] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:31:31,441] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:31:31,442] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:31:31,444] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:31:31,445] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:31:31,447] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:31:31,448] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:31:32,344] INFO : VALIDATION | Step=16000/500000 | Train Loss=3.9820 | Valid Loss=7.4646 | Train Loss Unweighted=0.9496 | Valid Loss Unweighted=1.3026 | Stale=14 | ETA=644.9m +rank0 [2026-11-15 03:31:32,345] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:31:32,468] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:31:32,488] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:31:32,538] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:31:32,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:31:33,145] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:31:33,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:31:33,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:31:33,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:31:33,757] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:31:33,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:31:34,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:31:34,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:31:34,368] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:31:34,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:31:34,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:31:34,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:31:34,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:31:35,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:31:35,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:31:35,337] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:31:35,347] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:31:35,700] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:31:35,985] INFO : Plotted number of elements +rank0 [2026-11-15 03:31:36,830] INFO : Plotted jets +rank0 [2026-11-15 03:31:38,044] INFO : Plotted jet ratio +rank0 [2026-11-15 03:31:59,439] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 03:31:59,439] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:32:40,552] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 03:32:40,552] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:32:51,212] INFO : Step 17000/500000 rank0 | Train Loss: 1.4746 | Train Loss Unweighted: 0.7059 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.6805 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0127 | Regression_eta: 0.0004 | Regression_pt: 0.0116 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.6805 | Regression_cos_phi: 0.0394 | Regression_energy: 0.1394 | Regression_eta: 0.1928 | Regression_pt: 0.1315 | Regression_sin_phi: 0.2557 | LR: 3.51e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8128s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0263 | max_optimizer_step: 0.0805 | max_tensorboard_logging: 0.2980 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3144.2 | uss_MB: 1948.7 | child_rss_MB: 23024.2 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:32:51,213] INFO : Running validation on rank0 +rank0 [2026-11-15 03:32:51,213] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:32:51,213] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:32:51,294] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 03:32:51,369] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:32:51,438] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:32:51,487] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:32:51,533] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:32:51,578] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:32:51,624] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:32:51,670] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:32:51,746] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:32:51,815] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:32:51,883] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:32:51,929] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:32:51,973] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:32:52,019] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:32:52,065] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:32:52,111] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:32:52,174] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:32:52,178] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:32:52,180] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:32:52,181] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:32:52,183] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:32:52,184] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:32:52,186] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:32:52,186] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:32:52,188] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:32:52,189] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:32:52,190] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:32:52,191] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:32:52,193] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:32:52,193] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:32:52,195] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:32:52,196] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:32:53,084] INFO : VALIDATION | Step=17000/500000 | Train Loss=1.4746 | Valid Loss=7.5702 | Train Loss Unweighted=0.7059 | Valid Loss Unweighted=1.3622 | Stale=15 | ETA=644.0m +rank0 [2026-11-15 03:32:53,084] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:32:53,210] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:32:53,231] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:32:53,285] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:32:53,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:32:53,861] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:32:54,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:32:54,164] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:32:54,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:32:54,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:32:54,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:32:54,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:32:54,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:32:55,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:32:55,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:32:55,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:32:55,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:32:55,692] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:32:55,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:32:55,999] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:32:56,074] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:32:56,086] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:32:56,446] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:32:56,704] INFO : Plotted number of elements +rank0 [2026-11-15 03:32:57,585] INFO : Plotted jets +rank0 [2026-11-15 03:32:58,873] INFO : Plotted jet ratio +rank0 [2026-11-15 03:33:29,810] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 03:33:29,810] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:34:12,630] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 03:34:12,630] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:34:14,524] INFO : Step 18000/500000 rank0 | Train Loss: 1.3724 | Train Loss Unweighted: 0.4318 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.3656 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0322 | Regression_eta: 0.0002 | Regression_pt: 0.0333 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0181 | Classification_binary: 0.3656 | Regression_cos_phi: 0.0707 | Regression_energy: 0.3547 | Regression_eta: 0.1141 | Regression_pt: 0.3782 | Regression_sin_phi: 0.0710 | LR: 3.69e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.8253s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0259 | max_optimizer_step: 0.0865 | max_tensorboard_logging: 0.2834 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3166.7 | uss_MB: 1971.2 | child_rss_MB: 23032.2 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:34:14,525] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 03:34:14,578] INFO : Running validation on rank0 +rank0 [2026-11-15 03:34:14,578] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:34:14,578] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:34:14,663] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 03:34:14,733] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:34:14,802] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:34:14,851] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:34:14,897] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:34:14,943] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:34:14,989] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:34:15,035] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:34:15,111] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:34:15,181] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:34:15,249] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:34:15,296] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:34:15,341] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:34:15,386] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:34:15,432] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:34:15,478] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:34:15,543] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:34:15,546] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:34:15,548] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:34:15,550] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:34:15,551] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:34:15,553] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:34:15,554] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:34:15,555] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:34:15,556] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:34:15,557] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:34:15,558] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:34:15,559] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:34:15,561] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:34:15,562] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:34:15,564] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:34:15,565] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:34:16,457] INFO : VALIDATION | Step=18000/500000 | Train Loss=1.3724 | Valid Loss=7.5679 | Train Loss Unweighted=0.4318 | Valid Loss Unweighted=1.4301 | Stale=16 | ETA=644.2m +rank0 [2026-11-15 03:34:16,457] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:34:16,589] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:34:16,610] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:34:16,659] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:34:17,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:34:17,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:34:17,421] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:34:17,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:34:17,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:34:17,887] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:34:18,043] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:34:18,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:34:18,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:34:18,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:34:18,666] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:34:18,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:34:18,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:34:19,132] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:34:19,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:34:19,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:34:19,516] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:34:19,527] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:34:19,890] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:34:20,155] INFO : Plotted number of elements +rank0 [2026-11-15 03:34:21,012] INFO : Plotted jets +rank0 [2026-11-15 03:34:22,236] INFO : Plotted jet ratio +rank0 [2026-11-15 03:35:01,166] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 03:35:01,166] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:35:33,924] INFO : Step 19000/500000 rank0 | Train Loss: 1.3276 | Train Loss Unweighted: 0.5230 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.4634 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0295 | Regression_eta: 0.0001 | Regression_pt: 0.0297 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0023 | Classification_binary: 0.4634 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3249 | Regression_eta: 0.0690 | Regression_pt: 0.3374 | Regression_sin_phi: 0.0829 | LR: 3.88e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8888s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0005 | max_forward: 0.0250 | max_loss: 0.0025 | max_backward: 0.0275 | max_optimizer_step: 0.0908 | max_tensorboard_logging: 0.2992 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3189.9 | uss_MB: 1994.4 | child_rss_MB: 23033.9 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:35:33,926] INFO : Running validation on rank0 +rank0 [2026-11-15 03:35:33,926] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:35:33,926] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:35:34,010] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 03:35:34,084] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:35:34,156] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:35:34,207] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:35:34,253] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:35:34,299] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:35:34,346] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:35:34,393] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:35:34,468] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:35:34,540] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:35:34,613] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:35:34,662] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:35:34,709] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:35:34,756] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:35:34,804] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:35:34,851] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:35:34,916] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:35:34,920] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:35:34,921] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:35:34,923] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:35:34,925] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:35:34,926] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:35:34,928] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:35:34,928] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:35:34,930] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:35:34,931] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:35:34,932] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:35:34,933] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:35:34,934] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:35:34,935] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:35:34,936] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:35:34,938] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:35:35,843] INFO : VALIDATION | Step=19000/500000 | Train Loss=1.3276 | Valid Loss=8.0248 | Train Loss Unweighted=0.5230 | Valid Loss Unweighted=1.4477 | Stale=17 | ETA=642.5m +rank0 [2026-11-15 03:35:35,844] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:35:35,974] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:35:35,994] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:35:36,047] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:35:36,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:35:36,641] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:35:36,796] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:35:36,953] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:35:37,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:35:37,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:35:37,418] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:35:37,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:35:37,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:35:37,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:35:38,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:35:38,200] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:35:38,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:35:38,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:35:38,661] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:35:38,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:35:38,903] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:35:38,913] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:35:39,276] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:35:39,537] INFO : Plotted number of elements +rank0 [2026-11-15 03:35:40,388] INFO : Plotted jets +rank0 [2026-11-15 03:35:41,650] INFO : Plotted jet ratio +rank0 [2026-11-15 03:35:49,881] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 03:35:49,881] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:36:30,689] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 03:36:30,689] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:36:55,689] INFO : Step 20000/500000 rank0 | Train Loss: 1.3270 | Train Loss Unweighted: 0.6699 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.6322 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0187 | Regression_eta: 0.0002 | Regression_pt: 0.0184 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0180 | Classification_binary: 0.6322 | Regression_cos_phi: 0.0484 | Regression_energy: 0.2058 | Regression_eta: 0.0903 | Regression_pt: 0.2086 | Regression_sin_phi: 0.1236 | LR: 4.07e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8894s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0261 | max_loss: 0.0026 | max_backward: 0.0268 | max_optimizer_step: 0.0886 | max_tensorboard_logging: 0.3148 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3140.9 | uss_MB: 1945.3 | child_rss_MB: 23033.9 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:36:55,691] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 03:36:55,744] INFO : Running validation on rank0 +rank0 [2026-11-15 03:36:55,744] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:36:55,744] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:36:55,837] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 03:36:55,922] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:36:55,994] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:36:56,044] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:36:56,095] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:36:56,145] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:36:56,194] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:36:56,252] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:36:56,337] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:36:56,416] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:36:56,491] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:36:56,541] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:36:56,591] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:36:56,640] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:36:56,690] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:36:56,742] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:36:56,813] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:36:56,817] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:36:56,819] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:36:56,821] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:36:56,822] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:36:56,824] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:36:56,826] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:36:56,827] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:36:56,829] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:36:56,830] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:36:56,831] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:36:56,832] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:36:56,834] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:36:56,834] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:36:56,836] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:36:56,837] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:36:57,816] INFO : VALIDATION | Step=20000/500000 | Train Loss=1.3270 | Valid Loss=8.3582 | Train Loss Unweighted=0.6699 | Valid Loss Unweighted=1.5498 | Stale=18 | ETA=641.9m +rank0 [2026-11-15 03:36:57,816] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:36:57,967] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:36:57,986] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:36:58,038] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:36:58,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:36:58,684] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:36:58,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:36:59,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:36:59,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:36:59,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:36:59,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:36:59,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:36:59,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:36:59,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:37:00,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:37:00,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:37:00,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:37:00,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:37:00,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:37:00,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:37:01,045] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:37:01,065] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:37:01,427] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:37:01,688] INFO : Plotted number of elements +rank0 [2026-11-15 03:37:02,539] INFO : Plotted jets +rank0 [2026-11-15 03:37:03,776] INFO : Plotted jet ratio +rank0 [2026-11-15 03:37:21,110] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 03:37:21,110] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:38:01,463] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 03:38:01,463] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:38:17,255] INFO : Step 21000/500000 rank0 | Train Loss: 5.1485 | Train Loss Unweighted: 2.1122 | Train Loss Components: Classification: 0.0001 | Classification_binary: 1.9020 | Regression_cos_phi: 0.0003 | Regression_energy: 0.1000 | Regression_eta: 0.0014 | Regression_pt: 0.1082 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0093 | Classification_binary: 1.9020 | Regression_cos_phi: 0.1159 | Regression_energy: 1.1022 | Regression_eta: 0.6547 | Regression_pt: 1.2301 | Regression_sin_phi: 0.1343 | LR: 4.28e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.9272s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0279 | max_loss: 0.0028 | max_backward: 0.0270 | max_optimizer_step: 0.0939 | max_tensorboard_logging: 0.3340 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3239.2 | uss_MB: 2043.7 | child_rss_MB: 23033.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 20.0 +rank0 [2026-11-15 03:38:17,257] INFO : Running validation on rank0 +rank0 [2026-11-15 03:38:17,257] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:38:17,257] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:38:17,351] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 03:38:17,432] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:38:17,505] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:38:17,554] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:38:17,602] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:38:17,649] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:38:17,698] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:38:17,747] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:38:17,826] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:38:17,898] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:38:17,969] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:38:18,016] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:38:18,065] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:38:18,117] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:38:18,170] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:38:18,218] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:38:18,289] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:38:18,293] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:38:18,295] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:38:18,297] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:38:18,298] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:38:18,300] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:38:18,301] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:38:18,303] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:38:18,305] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:38:18,306] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:38:18,307] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:38:18,308] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:38:18,310] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:38:18,310] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:38:18,311] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:38:18,313] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:38:19,315] INFO : VALIDATION | Step=21000/500000 | Train Loss=5.1485 | Valid Loss=8.5590 | Train Loss Unweighted=2.1122 | Valid Loss Unweighted=1.6248 | Stale=19 | ETA=641.0m +rank0 [2026-11-15 03:38:19,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:38:19,460] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:38:19,487] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:38:19,539] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:38:20,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:38:20,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:38:20,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:38:20,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:38:20,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:38:20,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:38:20,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:38:21,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:38:21,310] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:38:21,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:38:21,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:38:21,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:38:21,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:38:22,139] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:38:22,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:38:22,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:38:22,536] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:38:22,555] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:38:22,968] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:38:23,266] INFO : Plotted number of elements +rank0 [2026-11-15 03:38:24,255] INFO : Plotted jets +rank0 [2026-11-15 03:38:25,678] INFO : Plotted jet ratio +rank0 [2026-11-15 03:38:53,041] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 03:38:53,041] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:39:33,612] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 03:39:33,612] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:39:40,172] INFO : Step 22000/500000 rank0 | Train Loss: 1.9265 | Train Loss Unweighted: 0.7450 | Train Loss Components: Classification: 0.0008 | Classification_binary: 0.6608 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0410 | Regression_eta: 0.0002 | Regression_pt: 0.0420 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.6608 | Regression_cos_phi: 0.1133 | Regression_energy: 0.4514 | Regression_eta: 0.0727 | Regression_pt: 0.4772 | Regression_sin_phi: 0.0849 | LR: 4.50e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.8004s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0022 | max_backward: 0.0253 | max_optimizer_step: 0.0792 | max_tensorboard_logging: 0.2954 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3266.1 | uss_MB: 2070.6 | child_rss_MB: 23032.2 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:39:40,174] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 03:39:40,223] INFO : Running validation on rank0 +rank0 [2026-11-15 03:39:40,223] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:39:40,223] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:39:40,313] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 03:39:40,383] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:39:40,451] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:39:40,500] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:39:40,545] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:39:40,590] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:39:40,635] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:39:40,681] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:39:40,755] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:39:40,824] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:39:40,891] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:39:40,937] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:39:40,981] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:39:41,026] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:39:41,071] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:39:41,116] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:39:41,180] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:39:41,184] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:39:41,185] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:39:41,187] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:39:41,189] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:39:41,190] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:39:41,192] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:39:41,192] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:39:41,194] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:39:41,195] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:39:41,195] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:39:41,197] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:39:41,198] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:39:41,200] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:39:41,201] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:39:41,203] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:39:42,094] INFO : VALIDATION | Step=22000/500000 | Train Loss=1.9265 | Valid Loss=8.4074 | Train Loss Unweighted=0.7450 | Valid Loss Unweighted=1.6609 | Stale=20 | ETA=640.6m +rank0 [2026-11-15 03:39:42,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:39:42,208] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:39:42,224] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:39:42,271] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:39:42,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:39:42,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:39:42,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:39:43,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:39:43,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:39:43,450] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:39:43,600] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:39:43,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:39:43,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:39:44,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:39:44,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:39:44,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:39:44,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:39:44,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:39:44,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:39:44,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:39:45,037] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:39:45,056] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:39:45,415] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:39:45,674] INFO : Plotted number of elements +rank0 [2026-11-15 03:39:46,517] INFO : Plotted jets +rank0 [2026-11-15 03:39:47,675] INFO : Plotted jet ratio +rank0 [2026-11-15 03:40:21,657] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 03:40:21,657] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:40:59,257] INFO : Step 23000/500000 rank0 | Train Loss: 1.6042 | Train Loss Unweighted: 0.5246 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.4570 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0329 | Regression_eta: 0.0003 | Regression_pt: 0.0339 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0055 | Classification_binary: 0.4570 | Regression_cos_phi: 0.0717 | Regression_energy: 0.3627 | Regression_eta: 0.1575 | Regression_pt: 0.3853 | Regression_sin_phi: 0.1645 | LR: 4.73e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.8040s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0242 | max_loss: 0.0024 | max_backward: 0.0263 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.2583 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3223.0 | uss_MB: 2027.5 | child_rss_MB: 23030.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:40:59,258] INFO : Running validation on rank0 +rank0 [2026-11-15 03:40:59,258] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:40:59,258] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:40:59,339] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 03:40:59,413] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:40:59,483] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:40:59,533] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:40:59,579] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:40:59,625] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:40:59,672] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:40:59,720] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:40:59,796] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:40:59,866] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:40:59,935] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:40:59,982] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:41:00,028] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:41:00,074] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:41:00,120] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:41:00,166] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:41:00,232] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:41:00,236] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:41:00,237] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:41:00,239] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:41:00,241] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:41:00,242] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:41:00,244] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:41:00,245] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:41:00,247] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:41:00,248] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:41:00,249] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:41:00,250] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:41:00,252] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:41:00,252] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:41:00,253] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:41:00,255] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:41:01,154] INFO : VALIDATION | Step=23000/500000 | Train Loss=1.6042 | Valid Loss=8.8877 | Train Loss Unweighted=0.5246 | Valid Loss Unweighted=1.7887 | Stale=21 | ETA=638.8m +rank0 [2026-11-15 03:41:01,154] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:41:01,274] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:41:01,290] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:41:01,341] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:41:01,781] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:41:01,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:41:02,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:41:02,247] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:41:02,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:41:02,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:41:02,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:41:02,861] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:41:03,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:41:03,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:41:03,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:41:03,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:41:03,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:41:03,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:41:03,929] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:41:04,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:41:04,148] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:41:04,156] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:41:04,517] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:41:04,780] INFO : Plotted number of elements +rank0 [2026-11-15 03:41:05,628] INFO : Plotted jets +rank0 [2026-11-15 03:41:06,871] INFO : Plotted jet ratio +rank0 [2026-11-15 03:41:10,240] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 03:41:10,240] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:41:50,299] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 03:41:50,300] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:42:18,773] INFO : Step 24000/500000 rank0 | Train Loss: 1.2911 | Train Loss Unweighted: 0.4779 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.4409 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0180 | Regression_eta: 0.0004 | Regression_pt: 0.0180 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0325 | Classification_binary: 0.4409 | Regression_cos_phi: 0.0506 | Regression_energy: 0.1982 | Regression_eta: 0.1702 | Regression_pt: 0.2047 | Regression_sin_phi: 0.1941 | LR: 4.97e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7878s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0255 | max_optimizer_step: 0.0814 | max_tensorboard_logging: 0.2679 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3315.0 | uss_MB: 2119.4 | child_rss_MB: 23029.0 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:42:18,775] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 03:42:18,827] INFO : Running validation on rank0 +rank0 [2026-11-15 03:42:18,827] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:42:18,827] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:42:18,920] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 03:42:18,992] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:42:19,062] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:42:19,112] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:42:19,159] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:42:19,205] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:42:19,251] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:42:19,298] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:42:19,375] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:42:19,446] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:42:19,515] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:42:19,562] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:42:19,607] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:42:19,651] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:42:19,697] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:42:19,742] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:42:19,808] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:42:19,812] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:42:19,813] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:42:19,815] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:42:19,816] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:42:19,818] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:42:19,819] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:42:19,820] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:42:19,821] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:42:19,823] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:42:19,823] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:42:19,825] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:42:19,826] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:42:19,828] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:42:19,829] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:42:19,831] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:42:20,727] INFO : VALIDATION | Step=24000/500000 | Train Loss=1.2911 | Valid Loss=9.1555 | Train Loss Unweighted=0.4779 | Valid Loss Unweighted=1.8422 | Stale=22 | ETA=637.2m +rank0 [2026-11-15 03:42:20,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:42:20,851] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:42:20,866] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:42:20,913] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:42:21,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:42:21,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:42:21,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:42:21,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:42:21,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:42:22,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:42:22,260] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:42:22,414] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:42:22,570] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:42:22,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:42:22,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:42:23,029] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:42:23,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:42:23,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:42:23,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:42:23,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:42:23,804] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:42:23,812] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:42:24,165] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:42:24,429] INFO : Plotted number of elements +rank0 [2026-11-15 03:42:25,274] INFO : Plotted jets +rank0 [2026-11-15 03:42:26,441] INFO : Plotted jet ratio +rank0 [2026-11-15 03:42:38,615] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 03:42:38,615] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:43:19,600] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 03:43:19,601] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:43:39,995] INFO : Step 25000/500000 rank0 | Train Loss: 1.0831 | Train Loss Unweighted: 0.2828 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.2251 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0281 | Regression_eta: 0.0001 | Regression_pt: 0.0292 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0077 | Classification_binary: 0.2251 | Regression_cos_phi: 0.0615 | Regression_energy: 0.3096 | Regression_eta: 0.0565 | Regression_pt: 0.3314 | Regression_sin_phi: 0.0913 | LR: 5.22e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8356s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0022 | max_backward: 0.0260 | max_optimizer_step: 0.0868 | max_tensorboard_logging: 0.2891 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3338.2 | uss_MB: 2142.6 | child_rss_MB: 23021.5 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:43:39,996] INFO : Running validation on rank0 +rank0 [2026-11-15 03:43:39,997] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:43:39,997] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:43:40,079] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 03:43:40,150] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:43:40,220] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:43:40,270] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:43:40,317] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:43:40,362] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:43:40,408] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:43:40,455] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:43:40,531] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:43:40,602] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:43:40,672] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:43:40,720] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:43:40,765] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:43:40,810] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:43:40,856] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:43:40,902] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:43:40,967] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:43:40,971] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:43:40,972] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:43:40,974] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:43:40,976] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:43:40,977] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:43:40,979] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:43:40,979] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:43:40,981] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:43:40,985] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:43:40,985] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:43:40,987] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:43:40,988] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:43:40,989] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:43:40,990] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:43:40,992] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:43:41,896] INFO : VALIDATION | Step=25000/500000 | Train Loss=1.0831 | Valid Loss=9.2090 | Train Loss Unweighted=0.2828 | Valid Loss Unweighted=1.9892 | Stale=23 | ETA=636.1m +rank0 [2026-11-15 03:43:41,897] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:43:42,035] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:43:42,051] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:43:42,105] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:43:42,555] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:43:42,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:43:42,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:43:43,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:43:43,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:43:43,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:43:43,477] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:43:43,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:43:43,781] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:43:43,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:43:44,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:43:44,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:43:44,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:43:44,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:43:44,700] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:43:44,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:43:45,040] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:43:45,048] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:43:45,408] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:43:45,672] INFO : Plotted number of elements +rank0 [2026-11-15 03:43:46,525] INFO : Plotted jets +rank0 [2026-11-15 03:43:47,773] INFO : Plotted jet ratio +rank0 [2026-11-15 03:44:08,653] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 03:44:08,653] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:44:47,965] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 03:44:47,965] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:44:59,193] INFO : Step 26000/500000 rank0 | Train Loss: 0.3400 | Train Loss Unweighted: 0.0542 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0398 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0073 | Regression_eta: 0.0001 | Regression_pt: 0.0069 | Regression_sin_phi: 0.0000 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0000 | Classification_binary: 0.0398 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0806 | Regression_eta: 0.0480 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0456 | LR: 5.47e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8611s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0024 | max_backward: 0.0263 | max_optimizer_step: 0.0821 | max_tensorboard_logging: 0.3401 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3354.1 | uss_MB: 2158.5 | child_rss_MB: 23032.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:44:59,195] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 03:44:59,244] INFO : Running validation on rank0 +rank0 [2026-11-15 03:44:59,244] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:44:59,244] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:44:59,327] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 03:44:59,400] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:44:59,469] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:44:59,519] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:44:59,565] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:44:59,611] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:44:59,657] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:44:59,702] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:44:59,779] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:44:59,847] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:44:59,915] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:44:59,962] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:45:00,007] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:45:00,052] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:45:00,098] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:45:00,143] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:45:00,208] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:45:00,211] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:45:00,213] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:45:00,214] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:45:00,216] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:45:00,217] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:45:00,219] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:45:00,220] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:45:00,223] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:45:00,223] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:45:00,225] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:45:00,225] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:45:00,227] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:45:00,227] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:45:00,229] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:45:00,230] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:45:01,122] INFO : VALIDATION | Step=26000/500000 | Train Loss=0.3400 | Valid Loss=9.2750 | Train Loss Unweighted=0.0542 | Valid Loss Unweighted=2.0339 | Stale=24 | ETA=634.4m +rank0 [2026-11-15 03:45:01,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:45:01,249] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:45:01,265] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:45:01,374] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:45:01,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:45:02,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:45:02,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:45:02,329] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:45:02,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:45:02,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:45:02,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:45:02,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:45:03,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:45:03,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:45:03,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:45:03,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:45:03,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:45:03,859] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:45:04,014] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:45:04,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:45:04,221] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:45:04,228] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:45:04,588] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:45:04,847] INFO : Plotted number of elements +rank0 [2026-11-15 03:45:05,737] INFO : Plotted jets +rank0 [2026-11-15 03:45:07,347] INFO : Plotted jet ratio +rank0 [2026-11-15 03:45:38,077] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 03:45:38,078] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:46:18,546] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 03:46:18,546] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:46:20,960] INFO : Step 27000/500000 rank0 | Train Loss: 0.4126 | Train Loss Unweighted: 0.0427 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0189 | Regression_cos_phi: 0.0000 | Regression_energy: 0.0118 | Regression_eta: 0.0001 | Regression_pt: 0.0118 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.0189 | Regression_cos_phi: 0.0126 | Regression_energy: 0.1300 | Regression_eta: 0.0313 | Regression_pt: 0.1342 | Regression_sin_phi: 0.0855 | LR: 5.74e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8201s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0023 | max_backward: 0.0259 | max_optimizer_step: 0.0853 | max_tensorboard_logging: 0.2669 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3385.1 | uss_MB: 2189.6 | child_rss_MB: 23033.3 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:46:20,962] INFO : Running validation on rank0 +rank0 [2026-11-15 03:46:20,962] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:46:20,962] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:46:21,060] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 03:46:21,152] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:46:21,223] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:46:21,273] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:46:21,320] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:46:21,367] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:46:21,414] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:46:21,462] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:46:21,541] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:46:21,613] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:46:21,684] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:46:21,733] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:46:21,783] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:46:21,831] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:46:21,881] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:46:21,929] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:46:21,996] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:46:22,000] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:46:22,002] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:46:22,003] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:46:22,005] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:46:22,007] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:46:22,008] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:46:22,009] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:46:22,011] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:46:22,012] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:46:22,012] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:46:22,014] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:46:22,015] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:46:22,016] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:46:22,017] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:46:22,019] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:46:22,927] INFO : VALIDATION | Step=27000/500000 | Train Loss=0.4126 | Valid Loss=9.6160 | Train Loss Unweighted=0.0427 | Valid Loss Unweighted=2.1721 | Stale=25 | ETA=633.5m +rank0 [2026-11-15 03:46:22,928] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:46:23,054] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:46:23,070] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:46:23,122] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:46:23,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:46:23,794] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:46:23,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:46:24,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:46:24,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:46:24,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:46:24,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:46:24,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:46:24,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:46:25,098] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:46:25,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:46:25,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:46:25,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:46:25,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:46:25,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:46:26,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:46:26,120] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:46:26,128] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:46:26,518] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:46:26,846] INFO : Plotted number of elements +rank0 [2026-11-15 03:46:27,925] INFO : Plotted jets +rank0 [2026-11-15 03:46:29,476] INFO : Plotted jet ratio +rank0 [2026-11-15 03:47:07,847] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 03:47:07,847] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:47:41,275] INFO : Step 28000/500000 rank0 | Train Loss: 0.8055 | Train Loss Unweighted: 0.2805 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.2451 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0177 | Regression_eta: 0.0001 | Regression_pt: 0.0174 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0010 | Classification_binary: 0.2451 | Regression_cos_phi: 0.0490 | Regression_energy: 0.1953 | Regression_eta: 0.0439 | Regression_pt: 0.1973 | Regression_sin_phi: 0.0739 | LR: 6.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8295s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0248 | max_loss: 0.0024 | max_backward: 0.0263 | max_optimizer_step: 0.0912 | max_tensorboard_logging: 0.2581 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3414.1 | uss_MB: 2218.5 | child_rss_MB: 23033.9 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:47:41,277] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 03:47:41,328] INFO : Running validation on rank0 +rank0 [2026-11-15 03:47:41,328] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:47:41,328] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:47:41,409] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 03:47:41,486] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:47:41,556] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:47:41,607] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:47:41,654] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:47:41,699] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:47:41,746] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:47:41,793] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:47:41,870] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:47:41,940] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:47:42,010] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:47:42,056] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:47:42,102] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:47:42,148] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:47:42,194] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:47:42,244] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:47:42,310] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:47:42,314] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:47:42,316] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:47:42,317] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:47:42,319] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:47:42,320] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:47:42,322] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:47:42,322] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:47:42,324] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:47:42,325] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:47:42,326] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:47:42,327] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:47:42,329] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:47:42,330] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:47:42,332] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:47:42,333] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:47:43,232] INFO : VALIDATION | Step=28000/500000 | Train Loss=0.8055 | Valid Loss=9.3077 | Train Loss Unweighted=0.2805 | Valid Loss Unweighted=2.1454 | Stale=26 | ETA=632.2m +rank0 [2026-11-15 03:47:43,232] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:47:43,347] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:47:43,365] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:47:43,416] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:47:43,901] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:47:44,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:47:44,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:47:44,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:47:44,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:47:44,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:47:44,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:47:44,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:47:45,140] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:47:45,307] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:47:45,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:47:45,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:47:45,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:47:45,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:47:46,096] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:47:46,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:47:46,321] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:47:46,329] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:47:46,691] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:47:46,953] INFO : Plotted number of elements +rank0 [2026-11-15 03:47:47,822] INFO : Plotted jets +rank0 [2026-11-15 03:47:49,004] INFO : Plotted jet ratio +rank0 [2026-11-15 03:47:56,421] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 03:47:56,421] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:48:36,310] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 03:48:36,310] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:49:01,185] INFO : Step 29000/500000 rank0 | Train Loss: 0.6816 | Train Loss Unweighted: 0.0973 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.0686 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0142 | Regression_eta: 0.0002 | Regression_pt: 0.0135 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.0686 | Regression_cos_phi: 0.0398 | Regression_energy: 0.1568 | Regression_eta: 0.1105 | Regression_pt: 0.1539 | Regression_sin_phi: 0.1141 | LR: 6.29e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8509s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0022 | max_backward: 0.0268 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2981 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3435.8 | uss_MB: 2240.3 | child_rss_MB: 23033.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:49:01,187] INFO : Running validation on rank0 +rank0 [2026-11-15 03:49:01,187] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:49:01,187] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:49:01,270] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 03:49:01,343] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:49:01,414] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:49:01,463] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:49:01,510] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:49:01,555] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:49:01,601] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:49:01,648] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:49:01,726] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:49:01,798] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:49:01,869] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:49:01,916] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:49:01,961] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:49:02,008] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:49:02,054] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:49:02,100] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:49:02,165] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:49:02,169] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:49:02,171] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:49:02,172] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:49:02,174] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:49:02,175] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:49:02,177] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:49:02,178] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:49:02,179] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:49:02,180] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:49:02,181] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:49:02,182] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:49:02,184] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:49:02,185] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:49:02,187] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:49:02,188] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:49:03,090] INFO : VALIDATION | Step=29000/500000 | Train Loss=0.6816 | Valid Loss=9.8536 | Train Loss Unweighted=0.0973 | Valid Loss Unweighted=2.2811 | Stale=27 | ETA=630.7m +rank0 [2026-11-15 03:49:03,091] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:49:03,224] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:49:03,241] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:49:03,295] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:49:03,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:49:03,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:49:04,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:49:04,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:49:04,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:49:04,611] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:49:04,792] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:49:04,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:49:05,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:49:05,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:49:05,479] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:49:05,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:49:05,817] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:49:05,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:49:06,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:49:06,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:49:06,390] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:49:06,398] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:49:06,803] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:49:07,086] INFO : Plotted number of elements +rank0 [2026-11-15 03:49:08,109] INFO : Plotted jets +rank0 [2026-11-15 03:49:09,652] INFO : Plotted jet ratio +rank0 [2026-11-15 03:49:26,364] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 03:49:26,364] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:50:06,532] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 03:50:06,532] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:50:22,769] INFO : Step 30000/500000 rank0 | Train Loss: 1.0952 | Train Loss Unweighted: 0.2885 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.2559 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0155 | Regression_eta: 0.0004 | Regression_pt: 0.0162 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0018 | Classification_binary: 0.2559 | Regression_cos_phi: 0.0947 | Regression_energy: 0.1708 | Regression_eta: 0.1699 | Regression_pt: 0.1847 | Regression_sin_phi: 0.2174 | LR: 6.58e-05 | DataLoad Time: 0.0013s | Model Forward Time: 0.9805s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0004 | max_forward: 0.0355 | max_loss: 0.0031 | max_backward: 0.0275 | max_optimizer_step: 0.0976 | max_tensorboard_logging: 0.2963 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3455.0 | uss_MB: 2259.5 | child_rss_MB: 23033.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:50:22,771] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 03:50:22,824] INFO : Running validation on rank0 +rank0 [2026-11-15 03:50:22,824] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:50:22,824] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:50:22,906] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 03:50:22,986] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:50:23,059] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:50:23,110] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:50:23,160] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:50:23,209] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:50:23,258] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:50:23,307] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:50:23,387] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:50:23,461] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:50:23,533] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:50:23,581] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:50:23,629] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:50:23,677] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:50:23,725] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:50:23,774] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:50:23,841] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:50:23,845] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:50:23,846] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:50:23,848] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:50:23,850] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:50:23,851] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:50:23,853] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:50:23,853] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:50:23,854] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:50:23,856] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:50:23,856] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:50:23,858] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:50:23,859] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:50:23,860] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:50:23,861] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:50:23,863] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:50:24,893] INFO : VALIDATION | Step=30000/500000 | Train Loss=1.0952 | Valid Loss=9.9910 | Train Loss Unweighted=0.2885 | Valid Loss Unweighted=2.2803 | Stale=28 | ETA=629.7m +rank0 [2026-11-15 03:50:24,893] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:50:25,029] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:50:25,047] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:50:25,104] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:50:25,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:50:25,783] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:50:25,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:50:26,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:50:26,247] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:50:26,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:50:26,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:50:26,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:50:26,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:50:27,011] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:50:27,178] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:50:27,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:50:27,496] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:50:27,661] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:50:27,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:50:27,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:50:28,020] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:50:28,028] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:50:28,386] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:50:28,651] INFO : Plotted number of elements +rank0 [2026-11-15 03:50:29,520] INFO : Plotted jets +rank0 [2026-11-15 03:50:30,842] INFO : Plotted jet ratio +rank0 [2026-11-15 03:50:57,797] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 03:50:57,797] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:51:40,419] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 03:51:40,419] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:51:47,953] INFO : Step 31000/500000 rank0 | Train Loss: 0.7322 | Train Loss Unweighted: 0.0720 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.0376 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0171 | Regression_eta: 0.0003 | Regression_pt: 0.0167 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0051 | Classification_binary: 0.0376 | Regression_cos_phi: 0.0381 | Regression_energy: 0.1884 | Regression_eta: 0.1360 | Regression_pt: 0.1897 | Regression_sin_phi: 0.1373 | LR: 6.88e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8239s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0259 | max_optimizer_step: 0.0844 | max_tensorboard_logging: 0.2741 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3408.7 | uss_MB: 2213.2 | child_rss_MB: 23032.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:51:47,955] INFO : Running validation on rank0 +rank0 [2026-11-15 03:51:47,955] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:51:47,955] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:51:48,039] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 03:51:48,117] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:51:48,188] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:51:48,242] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:51:48,289] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:51:48,337] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:51:48,386] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:51:48,435] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:51:48,518] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:51:48,595] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:51:48,671] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:51:48,721] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:51:48,773] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:51:48,824] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:51:48,878] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:51:48,929] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:51:49,001] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:51:49,005] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:51:49,007] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:51:49,009] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:51:49,011] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:51:49,013] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:51:49,015] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:51:49,018] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:51:49,019] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:51:49,021] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:51:49,022] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:51:49,023] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:51:49,026] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:51:49,027] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:51:49,029] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:51:49,031] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:51:50,068] INFO : VALIDATION | Step=31000/500000 | Train Loss=0.7322 | Valid Loss=10.5751 | Train Loss Unweighted=0.0720 | Valid Loss Unweighted=2.4504 | Stale=29 | ETA=629.6m +rank0 [2026-11-15 03:51:50,068] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:51:50,206] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:51:50,222] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:51:50,276] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:51:50,770] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:51:50,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:51:51,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:51:51,235] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:51:51,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:51:51,541] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:51:51,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:51:51,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:51:52,002] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:51:52,156] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:51:52,310] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:51:52,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:51:52,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:51:52,768] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:51:52,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:51:53,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:51:53,153] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:51:53,160] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:51:53,523] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:51:53,784] INFO : Plotted number of elements +rank0 [2026-11-15 03:51:54,657] INFO : Plotted jets +rank0 [2026-11-15 03:51:55,932] INFO : Plotted jet ratio +rank0 [2026-11-15 03:52:29,639] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-15 03:52:29,640] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:53:07,425] INFO : Step 32000/500000 rank0 | Train Loss: 0.5597 | Train Loss Unweighted: 0.0735 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0484 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0122 | Regression_eta: 0.0001 | Regression_pt: 0.0124 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0008 | Classification_binary: 0.0484 | Regression_cos_phi: 0.0650 | Regression_energy: 0.1342 | Regression_eta: 0.0702 | Regression_pt: 0.1412 | Regression_sin_phi: 0.0997 | LR: 7.19e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8462s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0025 | max_backward: 0.0262 | max_optimizer_step: 0.0909 | max_tensorboard_logging: 0.2695 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3520.3 | uss_MB: 2324.8 | child_rss_MB: 23030.5 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:53:07,427] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-32000.pth +rank0 [2026-11-15 03:53:07,477] INFO : Running validation on rank0 +rank0 [2026-11-15 03:53:07,477] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:53:07,477] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:53:07,558] INFO : saved batch0_epoch32000.parquet +rank0 [2026-11-15 03:53:07,628] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:53:07,699] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:53:07,750] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:53:07,797] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:53:07,843] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:53:07,889] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:53:07,936] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:53:08,014] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:53:08,084] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:53:08,153] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:53:08,199] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:53:08,245] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:53:08,291] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:53:08,337] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:53:08,383] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:53:08,448] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:53:08,452] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:53:08,454] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:53:08,455] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:53:08,457] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:53:08,458] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:53:08,460] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:53:08,461] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:53:08,463] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:53:08,464] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:53:08,465] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:53:08,466] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:53:08,468] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:53:08,468] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:53:08,470] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:53:08,471] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:53:09,371] INFO : VALIDATION | Step=32000/500000 | Train Loss=0.5597 | Valid Loss=11.0309 | Train Loss Unweighted=0.0735 | Valid Loss Unweighted=2.7526 | Stale=30 | ETA=628.0m +rank0 [2026-11-15 03:53:09,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:53:09,487] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:53:09,505] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:53:09,558] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:53:10,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:53:10,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:53:10,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:53:10,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:53:10,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:53:10,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:53:10,988] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:53:11,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:53:11,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:53:11,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:53:11,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:53:11,792] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:53:11,955] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:53:12,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:53:12,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:53:12,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:53:12,510] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:53:12,518] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:53:12,919] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:53:13,206] INFO : Plotted number of elements +rank0 [2026-11-15 03:53:14,210] INFO : Plotted jets +rank0 [2026-11-15 03:53:15,669] INFO : Plotted jet ratio +rank0 [2026-11-15 03:53:16,152] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 03:53:16,152] INFO : Training completed. Total time on device 0: 43.050min diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..e98b055b496f06895619b4309960729cf18e44f7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/train.log.0 @@ -0,0 +1,2561 @@ +rank0 [2026-11-15 03:10:09,725] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 03:10:09,725] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 03:10:09,725] INFO : Instantiating model +rank0 [2026-11-15 03:10:09,763] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 03:10:09,763] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 03:10:09,764] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 03:10:09,764] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 03:10:09,764] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 03:10:09,764] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 03:10:09,764] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:10:09,766] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 03:10:09,766] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:10:09,768] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 03:10:09,768] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:10:09,770] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 03:10:09,770] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:10:09,772] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 03:10:09,772] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:10:09,774] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 03:10:09,774] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:10:09,776] INFO : Backbone initialization took 0.01s +rank0 [2026-11-15 03:10:09,776] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 03:10:09,776] INFO : Initializing output DNNs +rank0 [2026-11-15 03:10:09,792] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:10:09,795] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:10:09,800] INFO : Output DNNs initialization took 0.02s +rank0 [2026-11-15 03:10:09,800] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 03:10:09,800] INFO : jagged_attention enabled=True +rank0 [2026-11-15 03:10:09,800] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 03:10:09,801] INFO : task_queries enabled=True +rank0 [2026-11-15 03:10:09,801] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 03:10:09,801] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 03:10:09,801] INFO : classification_query parameters: 256 +rank0 [2026-11-15 03:10:09,801] INFO : regression_query parameters: 256 +rank0 [2026-11-15 03:10:09,801] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 03:10:09,801] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 03:10:09,801] INFO : output_mode=elementwise +rank0 [2026-11-15 03:10:09,801] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 03:10:09,801] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 03:10:09,801] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 03:10:09,801] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 03:10:09,801] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 03:10:09,801] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 03:10:09,801] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 03:10:09,801] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 03:10:09,802] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 03:10:09,802] INFO : MLPF __init__ done +rank0 [2026-11-15 03:10:09,802] INFO : Instantiated model +rank0 [2026-11-15 03:10:09,802] INFO : Moving model to device rank=0 +rank0 [2026-11-15 03:10:09,813] INFO : Moved model to device rank=0 +rank0 [2026-11-15 03:10:09,814] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 03:10:09,925] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 03:10:09,925] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 03:10:09,925] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 03:10:10,004] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 03:10:10,005] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 03:10:10,006] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 03:10:10,006] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 03:10:10,006] INFO : Total parameters: 3910930 +rank0 [2026-11-15 03:10:10,007] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 03:10:10,008] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509 +rank0 [2026-11-15 03:10:10,009] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509 +rank0 [2026-11-15 03:10:10,009] INFO : Getting dataloaders +rank0 [2026-11-15 03:10:10,009] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:10:12,276] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:10:12,350] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:10:12,402] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 03:10:12,463] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:10:12,508] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 03:10:12,577] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 03:10:12,613] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:10:12,669] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 03:10:12,718] INFO : train_dataset: cld_edm_z_tautau_pf, 898 +rank0 [2026-11-15 03:10:12,770] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:10:12,770] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 03:10:12,770] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 03:10:12,770] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-15 03:10:12,770] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:10:12,785] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,799] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,813] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,827] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,841] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,855] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,868] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,882] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,896] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,911] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:10:12,911] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 03:10:12,911] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 03:10:12,962] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:10:13,110] INFO : Got dataloaders +rank0 [2026-11-15 03:10:13,111] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-15 03:10:13,111] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-15 03:10:13,111] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 03:10:13,129] INFO : Creating train_iterator +rank0 [2026-11-15 03:10:13,130] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:10:13,171] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:10:13,364] INFO : Created train_iterator +rank0 [2026-11-15 03:10:21,908] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.5670582455396653, 'Classification': 0.018406680354382844, 'Regression_pt': 0.1378782992810011, 'Regression_eta': 0.0033446101847221144, 'Regression_sin_phi': 0.0015505722568195778, 'Regression_cos_phi': 0.0040240935087786055, 'Regression_energy': 0.14219819964841007}, weights={'Classification_binary': 1.0, 'Classification': 85.13529968261719, 'Regression_pt': 11.365517616271973, 'Regression_eta': 468.53240966796875, 'Regression_sin_phi': 1000.0, 'Regression_cos_phi': 389.4189453125, 'Regression_energy': 11.02023983001709} +rank0 [2026-11-15 03:10:54,281] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 03:10:54,281] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:11:25,579] INFO : Step 1000/500000 rank0 | Train Loss: 5.2682 | Train Loss Unweighted: 2.1068 | Train Loss Components: Classification: 0.0028 | Classification_binary: 1.9324 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0844 | Regression_eta: 0.0005 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.2354 | Classification_binary: 1.9324 | Regression_cos_phi: 0.1922 | Regression_energy: 0.9305 | Regression_eta: 0.2113 | Regression_pt: 0.9706 | Regression_sin_phi: 0.7958 | LR: 2.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8121s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0022 | max_backward: 0.0252 | max_optimizer_step: 0.0780 | max_tensorboard_logging: 0.3046 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2616.5 | uss_MB: 1406.4 | child_rss_MB: 23279.7 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:11:25,581] INFO : Running validation on rank0 +rank0 [2026-11-15 03:11:25,581] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:11:25,581] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:11:25,781] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 03:11:25,955] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:11:26,011] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:11:26,056] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:11:26,099] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:11:26,137] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:11:26,185] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:11:26,230] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:11:26,273] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:11:26,327] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:11:26,379] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:11:26,418] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:11:26,460] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:11:26,494] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:11:26,534] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:11:26,575] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:11:26,613] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:11:26,615] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:11:26,617] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:11:26,619] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:11:26,620] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:11:26,622] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:11:26,624] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:11:26,624] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:11:26,626] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:11:26,627] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:11:26,628] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:11:26,629] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:11:26,631] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:11:26,632] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:11:26,634] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:11:26,635] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:11:27,717] INFO : VALIDATION | Step=1000/500000 | Train Loss=5.2682 | Valid Loss=6.4280 | Train Loss Unweighted=2.1068 | Valid Loss Unweighted=1.2495 | Stale=0 | ETA=620.3m +rank0 [2026-11-15 03:11:27,717] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:11:27,859] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:11:27,877] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:11:27,930] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:11:28,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:11:28,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:11:29,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:11:29,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:11:29,394] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:11:29,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:11:29,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:11:29,845] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:11:29,994] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:11:30,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:11:30,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:11:30,448] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:11:30,599] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:11:30,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:11:30,900] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:11:31,038] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:11:31,109] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:11:31,117] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:11:31,476] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_1000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:11:32,390] INFO : Plotted number of elements +rank0 [2026-11-15 03:11:33,266] INFO : Plotted jets +rank0 [2026-11-15 03:11:34,435] INFO : Plotted jet ratio +rank0 [2026-11-15 03:11:43,468] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 03:11:43,468] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:12:22,878] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 03:12:22,878] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:12:45,471] INFO : Step 2000/500000 rank0 | Train Loss: 3.6655 | Train Loss Unweighted: 0.9498 | Train Loss Components: Classification: 0.0081 | Classification_binary: 0.7749 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0816 | Regression_eta: 0.0003 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.6921 | Classification_binary: 0.7749 | Regression_cos_phi: 0.0676 | Regression_energy: 0.8988 | Regression_eta: 0.1267 | Regression_pt: 0.9617 | Regression_sin_phi: 0.1436 | LR: 2.02e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7679s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0232 | max_loss: 0.0022 | max_backward: 0.0261 | max_optimizer_step: 0.0776 | max_tensorboard_logging: 0.2807 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2733.4 | uss_MB: 1537.7 | child_rss_MB: 23031.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:12:45,473] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 03:12:45,523] INFO : Running validation on rank0 +rank0 [2026-11-15 03:12:45,523] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:12:45,523] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:12:45,603] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 03:12:45,677] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:12:45,746] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:12:45,792] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:12:45,837] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:12:45,882] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:12:45,927] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:12:45,972] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:12:46,045] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:12:46,114] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:12:46,181] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:12:46,227] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:12:46,271] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:12:46,315] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:12:46,360] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:12:46,405] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:12:46,475] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:12:46,479] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:12:46,481] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:12:46,483] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:12:46,484] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:12:46,486] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:12:46,487] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:12:46,488] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:12:46,489] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:12:46,490] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:12:46,491] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:12:46,492] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:12:46,494] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:12:46,495] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:12:46,497] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:12:46,498] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:12:47,426] INFO : VALIDATION | Step=2000/500000 | Train Loss=3.6655 | Valid Loss=6.1666 | Train Loss Unweighted=0.9498 | Valid Loss Unweighted=1.2067 | Stale=0 | ETA=640.3m +rank0 [2026-11-15 03:12:47,427] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:12:47,562] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:12:47,579] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:12:47,623] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:12:48,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:12:48,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:12:48,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:12:48,478] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:12:48,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:12:48,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:12:48,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:12:49,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:12:49,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:12:49,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:12:49,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:12:49,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:12:49,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:12:50,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:12:50,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:12:50,304] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:12:50,373] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:12:50,381] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:12:50,736] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_2000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:12:50,992] INFO : Plotted number of elements +rank0 [2026-11-15 03:12:51,832] INFO : Plotted jets +rank0 [2026-11-15 03:12:53,006] INFO : Plotted jet ratio +rank0 [2026-11-15 03:13:10,966] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 03:13:10,967] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:13:51,600] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 03:13:51,600] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:14:06,066] INFO : Step 3000/500000 rank0 | Train Loss: 5.3269 | Train Loss Unweighted: 1.0812 | Train Loss Components: Classification: 0.0225 | Classification_binary: 0.9403 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0594 | Regression_eta: 0.0015 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 1.9142 | Classification_binary: 0.9403 | Regression_cos_phi: 0.1610 | Regression_energy: 0.6551 | Regression_eta: 0.6928 | Regression_pt: 0.6446 | Regression_sin_phi: 0.3189 | LR: 2.05e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8446s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0022 | max_backward: 0.0254 | max_optimizer_step: 0.0800 | max_tensorboard_logging: 0.3206 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2786.2 | uss_MB: 1590.5 | child_rss_MB: 23032.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:14:06,068] INFO : Running validation on rank0 +rank0 [2026-11-15 03:14:06,068] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:14:06,068] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:14:06,149] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 03:14:06,225] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:14:06,295] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:14:06,344] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:14:06,390] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:14:06,435] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:14:06,480] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:14:06,526] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:14:06,601] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:14:06,670] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:14:06,740] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:14:06,787] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:14:06,833] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:14:06,879] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:14:06,925] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:14:06,971] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:14:07,042] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:14:07,046] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:14:07,048] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:14:07,050] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:14:07,051] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:14:07,052] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:14:07,054] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:14:07,055] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:14:07,056] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:14:07,057] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:14:07,058] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:14:07,059] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:14:07,061] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:14:07,062] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:14:07,064] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:14:07,065] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:14:07,958] INFO : VALIDATION | Step=3000/500000 | Train Loss=5.3269 | Valid Loss=6.1924 | Train Loss Unweighted=1.0812 | Valid Loss Unweighted=1.2035 | Stale=1 | ETA=648.4m +rank0 [2026-11-15 03:14:07,958] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:14:08,103] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:14:08,118] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:14:08,167] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:14:08,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:14:08,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:14:08,906] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:14:09,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:14:09,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:14:09,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:14:09,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:14:09,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:14:09,831] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:14:09,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:14:10,140] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:14:10,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:14:10,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:14:10,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:14:10,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:14:10,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:14:10,982] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:14:10,989] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:14:11,345] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_3000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:14:11,604] INFO : Plotted number of elements +rank0 [2026-11-15 03:14:12,447] INFO : Plotted jets +rank0 [2026-11-15 03:14:13,649] INFO : Plotted jet ratio +rank0 [2026-11-15 03:14:40,187] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 03:14:40,187] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:15:20,140] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 03:15:20,141] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:15:25,730] INFO : Step 4000/500000 rank0 | Train Loss: 2.1774 | Train Loss Unweighted: 0.8873 | Train Loss Components: Classification: 0.0016 | Classification_binary: 0.8083 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0382 | Regression_eta: 0.0004 | Regression_pt: 0.0384 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.1343 | Classification_binary: 0.8083 | Regression_cos_phi: 0.1015 | Regression_energy: 0.4211 | Regression_eta: 0.1928 | Regression_pt: 0.4367 | Regression_sin_phi: 0.0826 | LR: 2.08e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.8123s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0022 | max_backward: 0.0252 | max_optimizer_step: 0.0769 | max_tensorboard_logging: 0.3222 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2816.7 | uss_MB: 1621.1 | child_rss_MB: 23031.2 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:15:25,732] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 03:15:25,780] INFO : Running validation on rank0 +rank0 [2026-11-15 03:15:25,780] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:15:25,780] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:15:25,859] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 03:15:25,929] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:15:25,998] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:15:26,045] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:15:26,090] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:15:26,134] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:15:26,180] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:15:26,224] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:15:26,298] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:15:26,366] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:15:26,433] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:15:26,479] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:15:26,524] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:15:26,568] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:15:26,613] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:15:26,658] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:15:26,729] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:15:26,733] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:15:26,734] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:15:26,736] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:15:26,737] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:15:26,739] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:15:26,740] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:15:26,741] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:15:26,742] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:15:26,744] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:15:26,744] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:15:26,745] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:15:26,747] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:15:26,748] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:15:26,750] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:15:26,751] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:15:27,638] INFO : VALIDATION | Step=4000/500000 | Train Loss=2.1774 | Valid Loss=6.4703 | Train Loss Unweighted=0.8873 | Valid Loss Unweighted=1.1973 | Stale=2 | ETA=650.0m +rank0 [2026-11-15 03:15:27,638] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:15:27,783] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:15:27,799] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:15:27,852] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:15:28,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:15:28,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:15:28,558] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:15:28,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:15:28,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:15:29,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:15:29,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:15:29,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:15:29,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:15:29,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:15:29,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:15:29,923] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:15:30,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:15:30,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:15:30,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:15:30,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:15:30,615] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:15:30,622] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:15:30,977] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_4000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:15:31,240] INFO : Plotted number of elements +rank0 [2026-11-15 03:15:32,114] INFO : Plotted jets +rank0 [2026-11-15 03:15:33,247] INFO : Plotted jet ratio +rank0 [2026-11-15 03:16:08,648] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 03:16:08,649] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:16:45,111] INFO : Step 5000/500000 rank0 | Train Loss: 7.9209 | Train Loss Unweighted: 2.1594 | Train Loss Components: Classification: 0.0084 | Classification_binary: 1.7412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1831 | Regression_eta: 0.0008 | Regression_pt: 0.2249 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.7150 | Classification_binary: 1.7412 | Regression_cos_phi: 0.2682 | Regression_energy: 2.0179 | Regression_eta: 0.3967 | Regression_pt: 2.5565 | Regression_sin_phi: 0.2255 | LR: 2.13e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8242s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0245 | max_loss: 0.0023 | max_backward: 0.0255 | max_optimizer_step: 0.0814 | max_tensorboard_logging: 0.2972 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2840.4 | uss_MB: 1644.8 | child_rss_MB: 23031.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:16:45,113] INFO : Running validation on rank0 +rank0 [2026-11-15 03:16:45,113] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:16:45,113] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:16:45,193] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 03:16:45,264] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:16:45,335] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:16:45,384] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:16:45,430] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:16:45,475] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:16:45,521] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:16:45,567] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:16:45,642] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:16:45,712] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:16:45,779] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:16:45,827] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:16:45,873] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:16:45,918] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:16:45,964] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:16:46,010] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:16:46,082] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:16:46,086] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:16:46,087] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:16:46,089] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:16:46,090] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:16:46,092] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:16:46,093] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:16:46,095] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:16:46,095] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:16:46,096] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:16:46,098] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:16:46,098] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:16:46,101] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:16:46,101] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:16:46,103] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:16:46,104] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:16:47,000] INFO : VALIDATION | Step=5000/500000 | Train Loss=7.9209 | Valid Loss=6.4137 | Train Loss Unweighted=2.1594 | Valid Loss Unweighted=1.1654 | Stale=3 | ETA=649.9m +rank0 [2026-11-15 03:16:47,000] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:16:47,132] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:16:47,150] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:16:47,197] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:16:47,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:16:47,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:16:47,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:16:48,105] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:16:48,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:16:48,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:16:48,570] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:16:48,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:16:48,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:16:49,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:16:49,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:16:49,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:16:49,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:16:49,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:16:49,830] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:16:49,971] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:16:50,096] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:16:50,118] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:16:50,475] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_5000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:16:50,741] INFO : Plotted number of elements +rank0 [2026-11-15 03:16:51,620] INFO : Plotted jets +rank0 [2026-11-15 03:16:52,836] INFO : Plotted jet ratio +rank0 [2026-11-15 03:16:57,319] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 03:16:57,319] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:17:37,532] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 03:17:37,533] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:18:05,396] INFO : Step 6000/500000 rank0 | Train Loss: 9.6798 | Train Loss Unweighted: 2.1215 | Train Loss Components: Classification: 0.0020 | Classification_binary: 1.9755 | Regression_cos_phi: 0.0011 | Regression_energy: 0.0687 | Regression_eta: 0.0015 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.1663 | Classification_binary: 1.9755 | Regression_cos_phi: 0.4187 | Regression_energy: 0.7566 | Regression_eta: 0.7096 | Regression_pt: 0.7709 | Regression_sin_phi: 4.8821 | LR: 2.19e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8482s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0024 | max_backward: 0.0280 | max_optimizer_step: 0.1018 | max_tensorboard_logging: 0.2721 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2859.8 | uss_MB: 1664.3 | child_rss_MB: 23030.0 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:18:05,398] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 03:18:05,447] INFO : Running validation on rank0 +rank0 [2026-11-15 03:18:05,447] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:18:05,447] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:18:05,527] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 03:18:05,601] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:18:05,669] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:18:05,716] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:18:05,762] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:18:05,808] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:18:05,853] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:18:05,901] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:18:05,980] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:18:06,053] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:18:06,125] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:18:06,175] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:18:06,221] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:18:06,266] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:18:06,313] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:18:06,359] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:18:06,431] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:18:06,435] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:18:06,437] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:18:06,439] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:18:06,440] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:18:06,442] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:18:06,443] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:18:06,444] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:18:06,445] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:18:06,447] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:18:06,447] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:18:06,448] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:18:06,450] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:18:06,452] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:18:06,453] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:18:06,455] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:18:07,352] INFO : VALIDATION | Step=6000/500000 | Train Loss=9.6798 | Valid Loss=6.6658 | Train Loss Unweighted=2.1215 | Valid Loss Unweighted=1.1755 | Stale=4 | ETA=650.7m +rank0 [2026-11-15 03:18:07,353] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:18:07,484] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:18:07,500] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:18:07,553] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:18:07,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:18:08,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:18:08,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:18:08,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:18:08,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:18:08,771] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:18:08,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:18:09,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:18:09,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:18:09,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:18:09,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:18:09,699] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:18:09,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:18:10,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:18:10,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:18:10,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:18:10,378] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:18:10,389] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:18:10,745] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_6000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:18:11,004] INFO : Plotted number of elements +rank0 [2026-11-15 03:18:11,867] INFO : Plotted jets +rank0 [2026-11-15 03:18:13,072] INFO : Plotted jet ratio +rank0 [2026-11-15 03:18:26,707] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 03:18:26,707] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:19:06,618] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 03:19:06,618] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:19:25,754] INFO : Step 7000/500000 rank0 | Train Loss: 1.8712 | Train Loss Unweighted: 0.3106 | Train Loss Components: Classification: 0.0043 | Classification_binary: 0.2320 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0365 | Regression_eta: 0.0004 | Regression_pt: 0.0369 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.3630 | Classification_binary: 0.2320 | Regression_cos_phi: 0.0717 | Regression_energy: 0.4027 | Regression_eta: 0.1764 | Regression_pt: 0.4199 | Regression_sin_phi: 0.2056 | LR: 2.26e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8459s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0025 | max_backward: 0.0268 | max_optimizer_step: 0.0829 | max_tensorboard_logging: 0.3124 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2885.4 | uss_MB: 1689.9 | child_rss_MB: 23028.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:19:25,755] INFO : Running validation on rank0 +rank0 [2026-11-15 03:19:25,755] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:19:25,755] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:19:25,837] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 03:19:25,908] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:19:25,978] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:19:26,028] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:19:26,074] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:19:26,120] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:19:26,167] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:19:26,214] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:19:26,289] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:19:26,361] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:19:26,430] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:19:26,477] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:19:26,523] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:19:26,569] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:19:26,615] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:19:26,661] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:19:26,734] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:19:26,738] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:19:26,740] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:19:26,741] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:19:26,743] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:19:26,744] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:19:26,746] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:19:26,746] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:19:26,748] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:19:26,749] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:19:26,750] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:19:26,751] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:19:26,753] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:19:26,754] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:19:26,756] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:19:26,757] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:19:27,654] INFO : VALIDATION | Step=7000/500000 | Train Loss=1.8712 | Valid Loss=6.3896 | Train Loss Unweighted=0.3106 | Valid Loss Unweighted=1.1662 | Stale=5 | ETA=650.9m +rank0 [2026-11-15 03:19:27,654] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:19:27,797] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:19:27,815] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:19:27,863] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:19:28,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:19:28,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:19:28,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:19:28,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:19:28,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:19:29,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:19:29,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:19:29,478] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:19:29,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:19:29,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:19:29,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:19:30,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:19:30,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:19:30,496] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:19:30,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:19:30,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:19:30,925] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:19:30,944] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:19:31,321] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_7000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:19:31,617] INFO : Plotted number of elements +rank0 [2026-11-15 03:19:32,636] INFO : Plotted jets +rank0 [2026-11-15 03:19:34,050] INFO : Plotted jet ratio +rank0 [2026-11-15 03:19:56,407] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 03:19:56,407] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:20:36,564] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 03:20:36,565] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:20:46,869] INFO : Step 8000/500000 rank0 | Train Loss: 6.9264 | Train Loss Unweighted: 1.2544 | Train Loss Components: Classification: 0.0199 | Classification_binary: 1.0996 | Regression_cos_phi: 0.0051 | Regression_energy: 0.0634 | Regression_eta: 0.0003 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 1.6922 | Classification_binary: 1.0996 | Regression_cos_phi: 1.9939 | Regression_energy: 0.6989 | Regression_eta: 0.1450 | Regression_pt: 0.7441 | Regression_sin_phi: 0.5527 | LR: 2.34e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8530s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0279 | max_loss: 0.0025 | max_backward: 0.0281 | max_optimizer_step: 0.0846 | max_tensorboard_logging: 0.3033 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2920.6 | uss_MB: 1725.1 | child_rss_MB: 23026.8 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:20:46,871] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 03:20:46,921] INFO : Running validation on rank0 +rank0 [2026-11-15 03:20:46,921] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:20:46,921] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:20:46,999] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 03:20:47,071] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:20:47,139] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:20:47,187] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:20:47,233] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:20:47,278] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:20:47,324] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:20:47,370] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:20:47,445] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:20:47,514] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:20:47,581] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:20:47,628] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:20:47,672] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:20:47,717] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:20:47,763] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:20:47,808] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:20:47,882] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:20:47,886] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:20:47,887] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:20:47,889] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:20:47,891] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:20:47,892] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:20:47,894] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:20:47,895] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:20:47,897] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:20:47,898] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:20:47,899] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:20:47,900] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:20:47,902] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:20:47,903] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:20:47,904] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:20:47,906] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:20:48,798] INFO : VALIDATION | Step=8000/500000 | Train Loss=6.9264 | Valid Loss=6.4667 | Train Loss Unweighted=1.2544 | Valid Loss Unweighted=1.1727 | Stale=6 | ETA=651.6m +rank0 [2026-11-15 03:20:48,798] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:20:48,936] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:20:48,951] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:48,998] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:20:49,419] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:20:49,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:20:49,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:20:49,881] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:20:50,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:20:50,186] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:20:50,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:20:50,492] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:20:50,645] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:20:50,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:20:50,952] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:20:51,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:20:51,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:20:51,408] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:20:51,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:20:51,699] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:20:51,807] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:20:51,814] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:52,176] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_8000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:20:52,439] INFO : Plotted number of elements +rank0 [2026-11-15 03:20:53,309] INFO : Plotted jets +rank0 [2026-11-15 03:20:54,646] INFO : Plotted jet ratio +rank0 [2026-11-15 03:21:26,011] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 03:21:26,011] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:22:06,466] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 03:22:06,466] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:22:07,880] INFO : Step 9000/500000 rank0 | Train Loss: 1.2291 | Train Loss Unweighted: 0.6159 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.5826 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0162 | Regression_eta: 0.0002 | Regression_pt: 0.0164 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0128 | Classification_binary: 0.5826 | Regression_cos_phi: 0.0829 | Regression_energy: 0.1790 | Regression_eta: 0.1141 | Regression_pt: 0.1868 | Regression_sin_phi: 0.0710 | LR: 2.43e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.8971s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0270 | max_optimizer_step: 0.0873 | max_tensorboard_logging: 0.3584 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2868.3 | uss_MB: 1672.8 | child_rss_MB: 23028.8 | allocated_MB: 170.0 | reserved_MB: 296.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:22:07,881] INFO : Running validation on rank0 +rank0 [2026-11-15 03:22:07,881] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:22:07,881] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:22:07,963] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 03:22:08,035] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:22:08,104] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:22:08,153] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:22:08,199] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:22:08,244] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:22:08,290] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:22:08,337] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:22:08,414] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:22:08,484] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:22:08,551] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:22:08,598] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:22:08,644] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:22:08,689] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:22:08,735] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:22:08,780] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:22:08,844] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:22:08,848] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:22:08,849] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:22:08,851] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:22:08,853] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:22:08,854] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:22:08,856] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:22:08,857] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:22:08,858] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:22:08,860] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:22:08,860] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:22:08,862] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:22:08,863] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:22:08,865] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:22:08,866] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:22:08,868] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:22:09,760] INFO : VALIDATION | Step=9000/500000 | Train Loss=1.2291 | Valid Loss=6.4177 | Train Loss Unweighted=0.6159 | Valid Loss Unweighted=1.1763 | Stale=7 | ETA=651.6m +rank0 [2026-11-15 03:22:09,761] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:22:09,920] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:22:09,935] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:09,984] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:22:10,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:22:10,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:22:10,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:22:10,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:22:11,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:22:11,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:22:11,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:22:11,595] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:22:11,759] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:22:11,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:22:12,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:22:12,258] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:22:12,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:22:12,578] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:22:12,737] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:22:12,906] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:22:12,986] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:22:12,994] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:13,392] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_9000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:22:13,682] INFO : Plotted number of elements +rank0 [2026-11-15 03:22:14,739] INFO : Plotted jets +rank0 [2026-11-15 03:22:16,178] INFO : Plotted jet ratio +rank0 [2026-11-15 03:22:55,212] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 03:22:55,212] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:23:27,412] INFO : Step 10000/500000 rank0 | Train Loss: 3.2697 | Train Loss Unweighted: 0.5968 | Train Loss Components: Classification: 0.0133 | Classification_binary: 0.5215 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0311 | Regression_eta: 0.0008 | Regression_pt: 0.0292 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 1.1292 | Classification_binary: 0.5215 | Regression_cos_phi: 0.2202 | Regression_energy: 0.3428 | Regression_eta: 0.3871 | Regression_pt: 0.3317 | Regression_sin_phi: 0.3373 | LR: 2.52e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8388s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0023 | max_backward: 0.0257 | max_optimizer_step: 0.0826 | max_tensorboard_logging: 0.3124 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2965.7 | uss_MB: 1770.2 | child_rss_MB: 23034.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:23:27,414] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 03:23:27,465] INFO : Running validation on rank0 +rank0 [2026-11-15 03:23:27,465] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:23:27,465] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:23:27,544] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 03:23:27,616] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:23:27,685] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:23:27,734] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:23:27,780] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:23:27,825] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:23:27,871] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:23:27,920] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:23:27,996] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:23:28,067] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:23:28,137] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:23:28,185] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:23:28,230] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:23:28,275] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:23:28,321] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:23:28,367] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:23:28,442] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:23:28,446] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:23:28,448] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:23:28,449] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:23:28,451] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:23:28,452] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:23:28,454] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:23:28,454] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:23:28,456] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:23:28,457] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:23:28,458] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:23:28,459] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:23:28,461] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:23:28,462] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:23:28,464] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:23:28,465] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:23:29,368] INFO : VALIDATION | Step=10000/500000 | Train Loss=3.2697 | Valid Loss=6.5776 | Train Loss Unweighted=0.5968 | Valid Loss Unweighted=1.1829 | Stale=8 | ETA=650.3m +rank0 [2026-11-15 03:23:29,368] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:23:29,512] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:23:29,527] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:23:29,578] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:23:30,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:23:30,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:23:30,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:23:30,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:23:30,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:23:30,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:23:30,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:23:31,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:23:31,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:23:31,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:23:31,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:23:31,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:23:31,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:23:31,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:23:32,137] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:23:32,275] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:23:32,386] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:23:32,393] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:23:32,743] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:23:33,003] INFO : Plotted number of elements +rank0 [2026-11-15 03:23:33,850] INFO : Plotted jets +rank0 [2026-11-15 03:23:35,079] INFO : Plotted jet ratio +rank0 [2026-11-15 03:23:43,674] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 03:23:43,674] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:24:24,149] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 03:24:24,149] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:24:48,037] INFO : Step 11000/500000 rank0 | Train Loss: 2.7025 | Train Loss Unweighted: 0.5553 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.4838 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0343 | Regression_eta: 0.0002 | Regression_pt: 0.0352 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0158 | Classification_binary: 0.4838 | Regression_cos_phi: 0.2055 | Regression_energy: 0.3785 | Regression_eta: 0.0923 | Regression_pt: 0.3999 | Regression_sin_phi: 1.1266 | LR: 2.63e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8432s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0838 | max_tensorboard_logging: 0.3208 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3000.8 | uss_MB: 1805.2 | child_rss_MB: 23033.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:24:48,038] INFO : Running validation on rank0 +rank0 [2026-11-15 03:24:48,038] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:24:48,038] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:24:48,118] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 03:24:48,190] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:24:48,260] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:24:48,310] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:24:48,355] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:24:48,400] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:24:48,446] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:24:48,493] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:24:48,568] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:24:48,637] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:24:48,705] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:24:48,752] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:24:48,798] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:24:48,843] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:24:48,889] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:24:48,934] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:24:49,008] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:24:49,012] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:24:49,014] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:24:49,016] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:24:49,017] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:24:49,018] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:24:49,020] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:24:49,020] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:24:49,022] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:24:49,023] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:24:49,024] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:24:49,025] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:24:49,027] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:24:49,028] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:24:49,029] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:24:49,031] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:24:49,915] INFO : VALIDATION | Step=11000/500000 | Train Loss=2.7025 | Valid Loss=6.7501 | Train Loss Unweighted=0.5553 | Valid Loss Unweighted=1.1630 | Stale=9 | ETA=649.6m +rank0 [2026-11-15 03:24:49,916] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:24:50,063] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:24:50,091] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:50,138] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:24:50,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:24:50,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:24:50,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:24:51,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:24:51,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:24:51,330] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:24:51,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:24:51,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:24:51,785] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:24:51,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:24:52,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:24:52,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:24:52,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:24:52,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:24:52,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:24:52,830] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:24:52,891] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:24:52,898] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:53,253] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_11000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:24:53,512] INFO : Plotted number of elements +rank0 [2026-11-15 03:24:54,356] INFO : Plotted jets +rank0 [2026-11-15 03:24:55,499] INFO : Plotted jet ratio +rank0 [2026-11-15 03:25:12,817] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 03:25:12,817] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:25:52,744] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 03:25:52,745] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:26:07,677] INFO : Step 12000/500000 rank0 | Train Loss: 1.5575 | Train Loss Unweighted: 0.7127 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.6898 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0102 | Regression_eta: 0.0003 | Regression_pt: 0.0117 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0040 | Classification_binary: 0.6898 | Regression_cos_phi: 0.1119 | Regression_energy: 0.1123 | Regression_eta: 0.1440 | Regression_pt: 0.1330 | Regression_sin_phi: 0.3627 | LR: 2.75e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7700s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0269 | max_optimizer_step: 0.0831 | max_tensorboard_logging: 0.2250 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3012.3 | uss_MB: 1816.8 | child_rss_MB: 23033.4 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:26:07,679] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 03:26:07,729] INFO : Running validation on rank0 +rank0 [2026-11-15 03:26:07,729] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:26:07,729] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:26:07,808] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 03:26:07,883] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:26:07,953] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:26:08,003] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:26:08,050] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:26:08,096] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:26:08,142] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:26:08,189] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:26:08,263] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:26:08,333] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:26:08,402] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:26:08,450] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:26:08,495] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:26:08,541] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:26:08,587] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:26:08,633] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:26:08,710] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:26:08,714] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:26:08,716] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:26:08,717] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:26:08,719] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:26:08,720] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:26:08,722] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:26:08,722] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:26:08,724] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:26:08,725] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:26:08,726] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:26:08,727] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:26:08,729] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:26:08,730] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:26:08,732] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:26:08,733] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:26:09,632] INFO : VALIDATION | Step=12000/500000 | Train Loss=1.5575 | Valid Loss=7.1893 | Train Loss Unweighted=0.7127 | Valid Loss Unweighted=1.1955 | Stale=10 | ETA=648.3m +rank0 [2026-11-15 03:26:09,632] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:26:09,736] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:26:09,752] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:09,802] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:26:10,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:26:10,408] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:26:10,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:26:10,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:26:10,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:26:11,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:26:11,173] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:26:11,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:26:11,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:26:11,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:26:11,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:26:11,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:26:12,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:26:12,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:26:12,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:26:12,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:26:12,646] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:26:12,653] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:13,004] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_12000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:26:13,265] INFO : Plotted number of elements +rank0 [2026-11-15 03:26:14,114] INFO : Plotted jets +rank0 [2026-11-15 03:26:15,341] INFO : Plotted jet ratio +rank0 [2026-11-15 03:26:41,987] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 03:26:41,988] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:27:22,339] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 03:27:22,339] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:27:28,462] INFO : Step 13000/500000 rank0 | Train Loss: 2.3960 | Train Loss Unweighted: 1.4400 | Train Loss Components: Classification: 0.0007 | Classification_binary: 1.3922 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0229 | Regression_eta: 0.0003 | Regression_pt: 0.0234 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 1.3922 | Regression_cos_phi: 0.1759 | Regression_energy: 0.2524 | Regression_eta: 0.1259 | Regression_pt: 0.2655 | Regression_sin_phi: 0.1252 | LR: 2.88e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7432s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0022 | max_backward: 0.0264 | max_optimizer_step: 0.0819 | max_tensorboard_logging: 0.2271 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3047.3 | uss_MB: 1851.8 | child_rss_MB: 23032.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:27:28,463] INFO : Running validation on rank0 +rank0 [2026-11-15 03:27:28,463] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:27:28,463] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:27:28,541] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 03:27:28,611] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:27:28,679] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:27:28,728] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:27:28,774] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:27:28,819] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:27:28,864] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:27:28,910] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:27:28,984] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:27:29,053] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:27:29,121] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:27:29,167] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:27:29,211] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:27:29,256] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:27:29,302] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:27:29,347] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:27:29,422] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:27:29,426] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:27:29,428] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:27:29,430] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:27:29,431] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:27:29,433] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:27:29,434] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:27:29,435] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:27:29,436] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:27:29,437] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:27:29,438] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:27:29,440] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:27:29,441] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:27:29,443] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:27:29,444] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:27:29,446] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:27:30,334] INFO : VALIDATION | Step=13000/500000 | Train Loss=2.3960 | Valid Loss=6.9363 | Train Loss Unweighted=1.4400 | Valid Loss Unweighted=1.2287 | Stale=11 | ETA=647.6m +rank0 [2026-11-15 03:27:30,334] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:27:30,440] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:27:30,455] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:27:30,500] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:27:30,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:27:31,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:27:31,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:27:31,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:27:31,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:27:31,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:27:31,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:27:31,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:27:32,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:27:32,276] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:27:32,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:27:32,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:27:32,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:27:32,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:27:33,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:27:33,173] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:27:33,239] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:27:33,246] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:27:33,607] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_13000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:27:33,868] INFO : Plotted number of elements +rank0 [2026-11-15 03:27:34,707] INFO : Plotted jets +rank0 [2026-11-15 03:27:35,843] INFO : Plotted jet ratio +rank0 [2026-11-15 03:28:11,265] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 03:28:11,265] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:28:48,883] INFO : Step 14000/500000 rank0 | Train Loss: 2.6940 | Train Loss Unweighted: 0.5470 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.3921 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0766 | Regression_eta: 0.0002 | Regression_pt: 0.0773 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0103 | Classification_binary: 0.3921 | Regression_cos_phi: 0.1141 | Regression_energy: 0.8443 | Regression_eta: 0.0928 | Regression_pt: 0.8780 | Regression_sin_phi: 0.3624 | LR: 3.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7707s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0022 | max_backward: 0.0268 | max_optimizer_step: 0.0855 | max_tensorboard_logging: 0.2315 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3066.5 | uss_MB: 1870.9 | child_rss_MB: 23031.7 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:28:48,891] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 03:28:48,940] INFO : Running validation on rank0 +rank0 [2026-11-15 03:28:48,940] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:28:48,940] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:28:49,022] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 03:28:49,097] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:28:49,166] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:28:49,216] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:28:49,262] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:28:49,308] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:28:49,354] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:28:49,401] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:28:49,477] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:28:49,553] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:28:49,623] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:28:49,669] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:28:49,715] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:28:49,760] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:28:49,807] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:28:49,853] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:28:49,930] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:28:49,935] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:28:49,936] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:28:49,938] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:28:49,939] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:28:49,941] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:28:49,942] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:28:49,944] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:28:49,945] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:28:49,947] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:28:49,947] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:28:49,949] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:28:49,950] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:28:49,952] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:28:49,953] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:28:49,955] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:28:50,850] INFO : VALIDATION | Step=14000/500000 | Train Loss=2.6940 | Valid Loss=7.2250 | Train Loss Unweighted=0.5470 | Valid Loss Unweighted=1.2046 | Stale=12 | ETA=646.7m +rank0 [2026-11-15 03:28:50,851] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:28:50,953] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:28:50,968] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:51,018] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:28:51,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:28:51,614] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:28:51,768] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:28:51,923] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:28:52,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:28:52,229] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:28:52,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:28:52,544] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:28:52,700] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:28:52,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:28:53,009] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:28:53,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:28:53,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:28:53,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:28:53,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:28:53,764] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:28:53,865] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:28:53,877] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:54,238] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_14000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:28:54,502] INFO : Plotted number of elements +rank0 [2026-11-15 03:28:55,363] INFO : Plotted jets +rank0 [2026-11-15 03:28:56,576] INFO : Plotted jet ratio +rank0 [2026-11-15 03:29:00,606] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 03:29:00,606] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:29:41,061] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 03:29:41,061] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:30:09,541] INFO : Step 15000/500000 rank0 | Train Loss: 1.7450 | Train Loss Unweighted: 0.8081 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.7603 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0232 | Regression_eta: 0.0001 | Regression_pt: 0.0237 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0023 | Classification_binary: 0.7603 | Regression_cos_phi: 0.2117 | Regression_energy: 0.2555 | Regression_eta: 0.0594 | Regression_pt: 0.2696 | Regression_sin_phi: 0.1862 | LR: 3.17e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7512s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0022 | max_backward: 0.0266 | max_optimizer_step: 0.0851 | max_tensorboard_logging: 0.2299 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3096.5 | uss_MB: 1901.0 | child_rss_MB: 23030.7 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:30:09,542] INFO : Running validation on rank0 +rank0 [2026-11-15 03:30:09,543] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:30:09,543] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:30:09,625] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 03:30:09,697] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:30:09,767] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:30:09,818] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:30:09,864] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:30:09,909] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:30:09,956] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:30:10,003] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:30:10,080] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:30:10,151] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:30:10,220] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:30:10,266] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:30:10,312] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:30:10,357] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:30:10,402] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:30:10,447] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:30:10,510] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:30:10,514] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:30:10,516] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:30:10,517] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:30:10,519] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:30:10,520] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:30:10,522] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:30:10,522] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:30:10,524] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:30:10,525] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:30:10,526] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:30:10,527] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:30:10,529] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:30:10,529] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:30:10,530] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:30:10,532] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:30:11,425] INFO : VALIDATION | Step=15000/500000 | Train Loss=1.7450 | Valid Loss=7.3469 | Train Loss Unweighted=0.8081 | Valid Loss Unweighted=1.2934 | Stale=13 | ETA=645.7m +rank0 [2026-11-15 03:30:11,426] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:30:11,531] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:30:11,551] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:11,596] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:30:12,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:30:12,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:30:12,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:30:12,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:30:12,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:30:12,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:30:12,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:30:13,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:30:13,245] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:30:13,399] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:30:13,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:30:13,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:30:13,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:30:14,011] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:30:14,164] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:30:14,303] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:30:14,372] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:30:14,382] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:14,745] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_15000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:30:15,006] INFO : Plotted number of elements +rank0 [2026-11-15 03:30:15,853] INFO : Plotted jets +rank0 [2026-11-15 03:30:17,011] INFO : Plotted jet ratio +rank0 [2026-11-15 03:30:29,505] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 03:30:29,505] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:31:10,566] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 03:31:10,566] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:31:30,404] INFO : Step 16000/500000 rank0 | Train Loss: 3.9820 | Train Loss Unweighted: 0.9496 | Train Loss Components: Classification: 0.0007 | Classification_binary: 0.7415 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1020 | Regression_eta: 0.0007 | Regression_pt: 0.1036 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.7415 | Regression_cos_phi: 0.3641 | Regression_energy: 1.1236 | Regression_eta: 0.3266 | Regression_pt: 1.1771 | Regression_sin_phi: 0.1891 | LR: 3.33e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.8017s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0023 | max_backward: 0.0257 | max_optimizer_step: 0.0829 | max_tensorboard_logging: 0.2689 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3121.3 | uss_MB: 1925.7 | child_rss_MB: 23028.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:31:30,412] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 03:31:30,462] INFO : Running validation on rank0 +rank0 [2026-11-15 03:31:30,462] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:31:30,462] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:31:30,544] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 03:31:30,616] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:31:30,686] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:31:30,735] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:31:30,782] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:31:30,827] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:31:30,873] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:31:30,919] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:31:30,994] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:31:31,065] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:31:31,133] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:31:31,180] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:31:31,225] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:31:31,270] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:31:31,316] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:31:31,361] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:31:31,426] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:31:31,430] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:31:31,431] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:31:31,433] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:31:31,434] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:31:31,436] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:31:31,437] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:31:31,438] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:31:31,439] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:31:31,441] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:31:31,441] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:31:31,442] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:31:31,444] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:31:31,445] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:31:31,447] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:31:31,448] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:31:32,344] INFO : VALIDATION | Step=16000/500000 | Train Loss=3.9820 | Valid Loss=7.4646 | Train Loss Unweighted=0.9496 | Valid Loss Unweighted=1.3026 | Stale=14 | ETA=644.9m +rank0 [2026-11-15 03:31:32,345] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:31:32,468] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:31:32,488] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:31:32,538] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:31:32,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:31:33,145] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:31:33,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:31:33,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:31:33,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:31:33,757] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:31:33,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:31:34,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:31:34,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:31:34,368] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:31:34,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:31:34,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:31:34,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:31:34,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:31:35,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:31:35,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:31:35,337] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:31:35,347] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:31:35,700] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_16000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:31:35,985] INFO : Plotted number of elements +rank0 [2026-11-15 03:31:36,830] INFO : Plotted jets +rank0 [2026-11-15 03:31:38,044] INFO : Plotted jet ratio +rank0 [2026-11-15 03:31:59,439] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 03:31:59,439] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:32:40,552] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 03:32:40,552] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:32:51,212] INFO : Step 17000/500000 rank0 | Train Loss: 1.4746 | Train Loss Unweighted: 0.7059 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.6805 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0127 | Regression_eta: 0.0004 | Regression_pt: 0.0116 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.6805 | Regression_cos_phi: 0.0394 | Regression_energy: 0.1394 | Regression_eta: 0.1928 | Regression_pt: 0.1315 | Regression_sin_phi: 0.2557 | LR: 3.51e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8128s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0263 | max_optimizer_step: 0.0805 | max_tensorboard_logging: 0.2980 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3144.2 | uss_MB: 1948.7 | child_rss_MB: 23024.2 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:32:51,213] INFO : Running validation on rank0 +rank0 [2026-11-15 03:32:51,213] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:32:51,213] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:32:51,294] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 03:32:51,369] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:32:51,438] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:32:51,487] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:32:51,533] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:32:51,578] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:32:51,624] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:32:51,670] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:32:51,746] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:32:51,815] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:32:51,883] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:32:51,929] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:32:51,973] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:32:52,019] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:32:52,065] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:32:52,111] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:32:52,174] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:32:52,178] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:32:52,180] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:32:52,181] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:32:52,183] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:32:52,184] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:32:52,186] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:32:52,186] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:32:52,188] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:32:52,189] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:32:52,190] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:32:52,191] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:32:52,193] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:32:52,193] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:32:52,195] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:32:52,196] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:32:53,084] INFO : VALIDATION | Step=17000/500000 | Train Loss=1.4746 | Valid Loss=7.5702 | Train Loss Unweighted=0.7059 | Valid Loss Unweighted=1.3622 | Stale=15 | ETA=644.0m +rank0 [2026-11-15 03:32:53,084] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:32:53,210] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:32:53,231] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:32:53,285] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:32:53,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:32:53,861] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:32:54,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:32:54,164] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:32:54,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:32:54,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:32:54,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:32:54,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:32:54,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:32:55,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:32:55,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:32:55,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:32:55,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:32:55,692] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:32:55,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:32:55,999] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:32:56,074] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:32:56,086] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:32:56,446] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_17000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:32:56,704] INFO : Plotted number of elements +rank0 [2026-11-15 03:32:57,585] INFO : Plotted jets +rank0 [2026-11-15 03:32:58,873] INFO : Plotted jet ratio +rank0 [2026-11-15 03:33:29,810] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 03:33:29,810] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:34:12,630] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 03:34:12,630] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:34:14,524] INFO : Step 18000/500000 rank0 | Train Loss: 1.3724 | Train Loss Unweighted: 0.4318 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.3656 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0322 | Regression_eta: 0.0002 | Regression_pt: 0.0333 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0181 | Classification_binary: 0.3656 | Regression_cos_phi: 0.0707 | Regression_energy: 0.3547 | Regression_eta: 0.1141 | Regression_pt: 0.3782 | Regression_sin_phi: 0.0710 | LR: 3.69e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.8253s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0259 | max_optimizer_step: 0.0865 | max_tensorboard_logging: 0.2834 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3166.7 | uss_MB: 1971.2 | child_rss_MB: 23032.2 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:34:14,525] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 03:34:14,578] INFO : Running validation on rank0 +rank0 [2026-11-15 03:34:14,578] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:34:14,578] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:34:14,663] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 03:34:14,733] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:34:14,802] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:34:14,851] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:34:14,897] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:34:14,943] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:34:14,989] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:34:15,035] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:34:15,111] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:34:15,181] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:34:15,249] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:34:15,296] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:34:15,341] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:34:15,386] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:34:15,432] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:34:15,478] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:34:15,543] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:34:15,546] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:34:15,548] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:34:15,550] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:34:15,551] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:34:15,553] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:34:15,554] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:34:15,555] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:34:15,556] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:34:15,557] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:34:15,558] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:34:15,559] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:34:15,561] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:34:15,562] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:34:15,564] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:34:15,565] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:34:16,457] INFO : VALIDATION | Step=18000/500000 | Train Loss=1.3724 | Valid Loss=7.5679 | Train Loss Unweighted=0.4318 | Valid Loss Unweighted=1.4301 | Stale=16 | ETA=644.2m +rank0 [2026-11-15 03:34:16,457] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:34:16,589] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:34:16,610] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:34:16,659] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:34:17,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:34:17,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:34:17,421] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:34:17,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:34:17,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:34:17,887] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:34:18,043] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:34:18,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:34:18,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:34:18,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:34:18,666] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:34:18,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:34:18,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:34:19,132] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:34:19,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:34:19,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:34:19,516] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:34:19,527] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:34:19,890] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_18000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:34:20,155] INFO : Plotted number of elements +rank0 [2026-11-15 03:34:21,012] INFO : Plotted jets +rank0 [2026-11-15 03:34:22,236] INFO : Plotted jet ratio +rank0 [2026-11-15 03:35:01,166] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 03:35:01,166] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:35:33,924] INFO : Step 19000/500000 rank0 | Train Loss: 1.3276 | Train Loss Unweighted: 0.5230 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.4634 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0295 | Regression_eta: 0.0001 | Regression_pt: 0.0297 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0023 | Classification_binary: 0.4634 | Regression_cos_phi: 0.0477 | Regression_energy: 0.3249 | Regression_eta: 0.0690 | Regression_pt: 0.3374 | Regression_sin_phi: 0.0829 | LR: 3.88e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8888s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0005 | max_forward: 0.0250 | max_loss: 0.0025 | max_backward: 0.0275 | max_optimizer_step: 0.0908 | max_tensorboard_logging: 0.2992 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3189.9 | uss_MB: 1994.4 | child_rss_MB: 23033.9 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:35:33,926] INFO : Running validation on rank0 +rank0 [2026-11-15 03:35:33,926] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:35:33,926] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:35:34,010] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 03:35:34,084] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:35:34,156] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:35:34,207] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:35:34,253] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:35:34,299] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:35:34,346] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:35:34,393] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:35:34,468] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:35:34,540] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:35:34,613] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:35:34,662] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:35:34,709] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:35:34,756] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:35:34,804] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:35:34,851] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:35:34,916] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:35:34,920] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:35:34,921] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:35:34,923] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:35:34,925] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:35:34,926] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:35:34,928] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:35:34,928] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:35:34,930] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:35:34,931] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:35:34,932] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:35:34,933] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:35:34,934] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:35:34,935] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:35:34,936] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:35:34,938] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:35:35,843] INFO : VALIDATION | Step=19000/500000 | Train Loss=1.3276 | Valid Loss=8.0248 | Train Loss Unweighted=0.5230 | Valid Loss Unweighted=1.4477 | Stale=17 | ETA=642.5m +rank0 [2026-11-15 03:35:35,844] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:35:35,974] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:35:35,994] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:35:36,047] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:35:36,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:35:36,641] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:35:36,796] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:35:36,953] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:35:37,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:35:37,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:35:37,418] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:35:37,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:35:37,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:35:37,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:35:38,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:35:38,200] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:35:38,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:35:38,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:35:38,661] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:35:38,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:35:38,903] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:35:38,913] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:35:39,276] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_19000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:35:39,537] INFO : Plotted number of elements +rank0 [2026-11-15 03:35:40,388] INFO : Plotted jets +rank0 [2026-11-15 03:35:41,650] INFO : Plotted jet ratio +rank0 [2026-11-15 03:35:49,881] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 03:35:49,881] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:36:30,689] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 03:36:30,689] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:36:55,689] INFO : Step 20000/500000 rank0 | Train Loss: 1.3270 | Train Loss Unweighted: 0.6699 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.6322 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0187 | Regression_eta: 0.0002 | Regression_pt: 0.0184 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0180 | Classification_binary: 0.6322 | Regression_cos_phi: 0.0484 | Regression_energy: 0.2058 | Regression_eta: 0.0903 | Regression_pt: 0.2086 | Regression_sin_phi: 0.1236 | LR: 4.07e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8894s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0261 | max_loss: 0.0026 | max_backward: 0.0268 | max_optimizer_step: 0.0886 | max_tensorboard_logging: 0.3148 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3140.9 | uss_MB: 1945.3 | child_rss_MB: 23033.9 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:36:55,691] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 03:36:55,744] INFO : Running validation on rank0 +rank0 [2026-11-15 03:36:55,744] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:36:55,744] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:36:55,837] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 03:36:55,922] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:36:55,994] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:36:56,044] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:36:56,095] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:36:56,145] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:36:56,194] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:36:56,252] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:36:56,337] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:36:56,416] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:36:56,491] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:36:56,541] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:36:56,591] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:36:56,640] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:36:56,690] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:36:56,742] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:36:56,813] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:36:56,817] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:36:56,819] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:36:56,821] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:36:56,822] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:36:56,824] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:36:56,826] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:36:56,827] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:36:56,829] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:36:56,830] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:36:56,831] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:36:56,832] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:36:56,834] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:36:56,834] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:36:56,836] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:36:56,837] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:36:57,816] INFO : VALIDATION | Step=20000/500000 | Train Loss=1.3270 | Valid Loss=8.3582 | Train Loss Unweighted=0.6699 | Valid Loss Unweighted=1.5498 | Stale=18 | ETA=641.9m +rank0 [2026-11-15 03:36:57,816] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:36:57,967] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:36:57,986] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:36:58,038] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:36:58,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:36:58,684] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:36:58,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:36:59,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:36:59,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:36:59,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:36:59,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:36:59,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:36:59,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:36:59,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:37:00,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:37:00,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:37:00,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:37:00,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:37:00,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:37:00,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:37:01,045] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:37:01,065] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:37:01,427] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:37:01,688] INFO : Plotted number of elements +rank0 [2026-11-15 03:37:02,539] INFO : Plotted jets +rank0 [2026-11-15 03:37:03,776] INFO : Plotted jet ratio +rank0 [2026-11-15 03:37:21,110] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 03:37:21,110] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:38:01,463] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 03:38:01,463] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:38:17,255] INFO : Step 21000/500000 rank0 | Train Loss: 5.1485 | Train Loss Unweighted: 2.1122 | Train Loss Components: Classification: 0.0001 | Classification_binary: 1.9020 | Regression_cos_phi: 0.0003 | Regression_energy: 0.1000 | Regression_eta: 0.0014 | Regression_pt: 0.1082 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0093 | Classification_binary: 1.9020 | Regression_cos_phi: 0.1159 | Regression_energy: 1.1022 | Regression_eta: 0.6547 | Regression_pt: 1.2301 | Regression_sin_phi: 0.1343 | LR: 4.28e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.9272s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0279 | max_loss: 0.0028 | max_backward: 0.0270 | max_optimizer_step: 0.0939 | max_tensorboard_logging: 0.3340 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3239.2 | uss_MB: 2043.7 | child_rss_MB: 23033.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 20.0 +rank0 [2026-11-15 03:38:17,257] INFO : Running validation on rank0 +rank0 [2026-11-15 03:38:17,257] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:38:17,257] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:38:17,351] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 03:38:17,432] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:38:17,505] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:38:17,554] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:38:17,602] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:38:17,649] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:38:17,698] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:38:17,747] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:38:17,826] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:38:17,898] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:38:17,969] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:38:18,016] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:38:18,065] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:38:18,117] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:38:18,170] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:38:18,218] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:38:18,289] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:38:18,293] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:38:18,295] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:38:18,297] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:38:18,298] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:38:18,300] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:38:18,301] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:38:18,303] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:38:18,305] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:38:18,306] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:38:18,307] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:38:18,308] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:38:18,310] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:38:18,310] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:38:18,311] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:38:18,313] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:38:19,315] INFO : VALIDATION | Step=21000/500000 | Train Loss=5.1485 | Valid Loss=8.5590 | Train Loss Unweighted=2.1122 | Valid Loss Unweighted=1.6248 | Stale=19 | ETA=641.0m +rank0 [2026-11-15 03:38:19,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:38:19,460] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:38:19,487] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:38:19,539] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:38:20,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:38:20,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:38:20,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:38:20,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:38:20,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:38:20,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:38:20,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:38:21,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:38:21,310] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:38:21,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:38:21,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:38:21,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:38:21,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:38:22,139] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:38:22,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:38:22,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:38:22,536] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:38:22,555] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:38:22,968] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_21000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:38:23,266] INFO : Plotted number of elements +rank0 [2026-11-15 03:38:24,255] INFO : Plotted jets +rank0 [2026-11-15 03:38:25,678] INFO : Plotted jet ratio +rank0 [2026-11-15 03:38:53,041] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 03:38:53,041] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:39:33,612] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 03:39:33,612] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:39:40,172] INFO : Step 22000/500000 rank0 | Train Loss: 1.9265 | Train Loss Unweighted: 0.7450 | Train Loss Components: Classification: 0.0008 | Classification_binary: 0.6608 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0410 | Regression_eta: 0.0002 | Regression_pt: 0.0420 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.6608 | Regression_cos_phi: 0.1133 | Regression_energy: 0.4514 | Regression_eta: 0.0727 | Regression_pt: 0.4772 | Regression_sin_phi: 0.0849 | LR: 4.50e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.8004s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0022 | max_backward: 0.0253 | max_optimizer_step: 0.0792 | max_tensorboard_logging: 0.2954 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3266.1 | uss_MB: 2070.6 | child_rss_MB: 23032.2 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:39:40,174] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 03:39:40,223] INFO : Running validation on rank0 +rank0 [2026-11-15 03:39:40,223] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:39:40,223] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:39:40,313] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 03:39:40,383] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:39:40,451] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:39:40,500] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:39:40,545] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:39:40,590] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:39:40,635] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:39:40,681] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:39:40,755] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:39:40,824] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:39:40,891] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:39:40,937] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:39:40,981] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:39:41,026] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:39:41,071] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:39:41,116] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:39:41,180] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:39:41,184] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:39:41,185] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:39:41,187] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:39:41,189] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:39:41,190] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:39:41,192] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:39:41,192] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:39:41,194] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:39:41,195] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:39:41,195] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:39:41,197] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:39:41,198] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:39:41,200] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:39:41,201] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:39:41,203] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:39:42,094] INFO : VALIDATION | Step=22000/500000 | Train Loss=1.9265 | Valid Loss=8.4074 | Train Loss Unweighted=0.7450 | Valid Loss Unweighted=1.6609 | Stale=20 | ETA=640.6m +rank0 [2026-11-15 03:39:42,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:39:42,208] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:39:42,224] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:39:42,271] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:39:42,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:39:42,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:39:42,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:39:43,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:39:43,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:39:43,450] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:39:43,600] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:39:43,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:39:43,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:39:44,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:39:44,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:39:44,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:39:44,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:39:44,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:39:44,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:39:44,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:39:45,037] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:39:45,056] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:39:45,415] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_22000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:39:45,674] INFO : Plotted number of elements +rank0 [2026-11-15 03:39:46,517] INFO : Plotted jets +rank0 [2026-11-15 03:39:47,675] INFO : Plotted jet ratio +rank0 [2026-11-15 03:40:21,657] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 03:40:21,657] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:40:59,257] INFO : Step 23000/500000 rank0 | Train Loss: 1.6042 | Train Loss Unweighted: 0.5246 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.4570 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0329 | Regression_eta: 0.0003 | Regression_pt: 0.0339 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0055 | Classification_binary: 0.4570 | Regression_cos_phi: 0.0717 | Regression_energy: 0.3627 | Regression_eta: 0.1575 | Regression_pt: 0.3853 | Regression_sin_phi: 0.1645 | LR: 4.73e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.8040s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0242 | max_loss: 0.0024 | max_backward: 0.0263 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.2583 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3223.0 | uss_MB: 2027.5 | child_rss_MB: 23030.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:40:59,258] INFO : Running validation on rank0 +rank0 [2026-11-15 03:40:59,258] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:40:59,258] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:40:59,339] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 03:40:59,413] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:40:59,483] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:40:59,533] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:40:59,579] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:40:59,625] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:40:59,672] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:40:59,720] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:40:59,796] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:40:59,866] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:40:59,935] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:40:59,982] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:41:00,028] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:41:00,074] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:41:00,120] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:41:00,166] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:41:00,232] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:41:00,236] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:41:00,237] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:41:00,239] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:41:00,241] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:41:00,242] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:41:00,244] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:41:00,245] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:41:00,247] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:41:00,248] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:41:00,249] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:41:00,250] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:41:00,252] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:41:00,252] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:41:00,253] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:41:00,255] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:41:01,154] INFO : VALIDATION | Step=23000/500000 | Train Loss=1.6042 | Valid Loss=8.8877 | Train Loss Unweighted=0.5246 | Valid Loss Unweighted=1.7887 | Stale=21 | ETA=638.8m +rank0 [2026-11-15 03:41:01,154] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:41:01,274] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:41:01,290] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:41:01,341] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:41:01,781] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:41:01,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:41:02,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:41:02,247] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:41:02,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:41:02,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:41:02,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:41:02,861] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:41:03,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:41:03,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:41:03,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:41:03,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:41:03,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:41:03,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:41:03,929] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:41:04,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:41:04,148] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:41:04,156] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:41:04,517] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_23000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:41:04,780] INFO : Plotted number of elements +rank0 [2026-11-15 03:41:05,628] INFO : Plotted jets +rank0 [2026-11-15 03:41:06,871] INFO : Plotted jet ratio +rank0 [2026-11-15 03:41:10,240] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 03:41:10,240] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:41:50,299] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 03:41:50,300] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:42:18,773] INFO : Step 24000/500000 rank0 | Train Loss: 1.2911 | Train Loss Unweighted: 0.4779 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.4409 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0180 | Regression_eta: 0.0004 | Regression_pt: 0.0180 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0325 | Classification_binary: 0.4409 | Regression_cos_phi: 0.0506 | Regression_energy: 0.1982 | Regression_eta: 0.1702 | Regression_pt: 0.2047 | Regression_sin_phi: 0.1941 | LR: 4.97e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7878s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0255 | max_optimizer_step: 0.0814 | max_tensorboard_logging: 0.2679 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3315.0 | uss_MB: 2119.4 | child_rss_MB: 23029.0 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:42:18,775] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 03:42:18,827] INFO : Running validation on rank0 +rank0 [2026-11-15 03:42:18,827] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:42:18,827] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:42:18,920] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 03:42:18,992] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:42:19,062] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:42:19,112] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:42:19,159] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:42:19,205] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:42:19,251] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:42:19,298] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:42:19,375] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:42:19,446] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:42:19,515] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:42:19,562] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:42:19,607] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:42:19,651] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:42:19,697] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:42:19,742] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:42:19,808] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:42:19,812] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:42:19,813] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:42:19,815] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:42:19,816] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:42:19,818] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:42:19,819] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:42:19,820] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:42:19,821] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:42:19,823] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:42:19,823] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:42:19,825] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:42:19,826] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:42:19,828] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:42:19,829] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:42:19,831] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:42:20,727] INFO : VALIDATION | Step=24000/500000 | Train Loss=1.2911 | Valid Loss=9.1555 | Train Loss Unweighted=0.4779 | Valid Loss Unweighted=1.8422 | Stale=22 | ETA=637.2m +rank0 [2026-11-15 03:42:20,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:42:20,851] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:42:20,866] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:42:20,913] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:42:21,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:42:21,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:42:21,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:42:21,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:42:21,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:42:22,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:42:22,260] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:42:22,414] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:42:22,570] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:42:22,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:42:22,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:42:23,029] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:42:23,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:42:23,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:42:23,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:42:23,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:42:23,804] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:42:23,812] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:42:24,165] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_24000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:42:24,429] INFO : Plotted number of elements +rank0 [2026-11-15 03:42:25,274] INFO : Plotted jets +rank0 [2026-11-15 03:42:26,441] INFO : Plotted jet ratio +rank0 [2026-11-15 03:42:38,615] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 03:42:38,615] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:43:19,600] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 03:43:19,601] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:43:39,995] INFO : Step 25000/500000 rank0 | Train Loss: 1.0831 | Train Loss Unweighted: 0.2828 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.2251 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0281 | Regression_eta: 0.0001 | Regression_pt: 0.0292 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0077 | Classification_binary: 0.2251 | Regression_cos_phi: 0.0615 | Regression_energy: 0.3096 | Regression_eta: 0.0565 | Regression_pt: 0.3314 | Regression_sin_phi: 0.0913 | LR: 5.22e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8356s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0022 | max_backward: 0.0260 | max_optimizer_step: 0.0868 | max_tensorboard_logging: 0.2891 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3338.2 | uss_MB: 2142.6 | child_rss_MB: 23021.5 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:43:39,996] INFO : Running validation on rank0 +rank0 [2026-11-15 03:43:39,997] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:43:39,997] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:43:40,079] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 03:43:40,150] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:43:40,220] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:43:40,270] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:43:40,317] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:43:40,362] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:43:40,408] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:43:40,455] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:43:40,531] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:43:40,602] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:43:40,672] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:43:40,720] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:43:40,765] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:43:40,810] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:43:40,856] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:43:40,902] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:43:40,967] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:43:40,971] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:43:40,972] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:43:40,974] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:43:40,976] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:43:40,977] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:43:40,979] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:43:40,979] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:43:40,981] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:43:40,985] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:43:40,985] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:43:40,987] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:43:40,988] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:43:40,989] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:43:40,990] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:43:40,992] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:43:41,896] INFO : VALIDATION | Step=25000/500000 | Train Loss=1.0831 | Valid Loss=9.2090 | Train Loss Unweighted=0.2828 | Valid Loss Unweighted=1.9892 | Stale=23 | ETA=636.1m +rank0 [2026-11-15 03:43:41,897] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:43:42,035] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:43:42,051] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:43:42,105] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:43:42,555] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:43:42,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:43:42,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:43:43,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:43:43,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:43:43,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:43:43,477] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:43:43,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:43:43,781] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:43:43,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:43:44,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:43:44,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:43:44,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:43:44,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:43:44,700] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:43:44,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:43:45,040] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:43:45,048] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:43:45,408] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_25000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:43:45,672] INFO : Plotted number of elements +rank0 [2026-11-15 03:43:46,525] INFO : Plotted jets +rank0 [2026-11-15 03:43:47,773] INFO : Plotted jet ratio +rank0 [2026-11-15 03:44:08,653] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 03:44:08,653] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:44:47,965] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 03:44:47,965] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:44:59,193] INFO : Step 26000/500000 rank0 | Train Loss: 0.3400 | Train Loss Unweighted: 0.0542 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0398 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0073 | Regression_eta: 0.0001 | Regression_pt: 0.0069 | Regression_sin_phi: 0.0000 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0000 | Classification_binary: 0.0398 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0806 | Regression_eta: 0.0480 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0456 | LR: 5.47e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8611s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0024 | max_backward: 0.0263 | max_optimizer_step: 0.0821 | max_tensorboard_logging: 0.3401 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3354.1 | uss_MB: 2158.5 | child_rss_MB: 23032.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:44:59,195] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 03:44:59,244] INFO : Running validation on rank0 +rank0 [2026-11-15 03:44:59,244] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:44:59,244] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:44:59,327] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 03:44:59,400] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:44:59,469] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:44:59,519] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:44:59,565] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:44:59,611] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:44:59,657] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:44:59,702] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:44:59,779] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:44:59,847] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:44:59,915] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:44:59,962] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:45:00,007] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:45:00,052] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:45:00,098] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:45:00,143] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:45:00,208] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:45:00,211] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:45:00,213] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:45:00,214] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:45:00,216] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:45:00,217] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:45:00,219] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:45:00,220] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:45:00,223] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:45:00,223] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:45:00,225] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:45:00,225] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:45:00,227] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:45:00,227] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:45:00,229] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:45:00,230] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:45:01,122] INFO : VALIDATION | Step=26000/500000 | Train Loss=0.3400 | Valid Loss=9.2750 | Train Loss Unweighted=0.0542 | Valid Loss Unweighted=2.0339 | Stale=24 | ETA=634.4m +rank0 [2026-11-15 03:45:01,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:45:01,249] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:45:01,265] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:45:01,374] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:45:01,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:45:02,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:45:02,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:45:02,329] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:45:02,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:45:02,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:45:02,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:45:02,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:45:03,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:45:03,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:45:03,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:45:03,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:45:03,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:45:03,859] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:45:04,014] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:45:04,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:45:04,221] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:45:04,228] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:45:04,588] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_26000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:45:04,847] INFO : Plotted number of elements +rank0 [2026-11-15 03:45:05,737] INFO : Plotted jets +rank0 [2026-11-15 03:45:07,347] INFO : Plotted jet ratio +rank0 [2026-11-15 03:45:38,077] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 03:45:38,078] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:46:18,546] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 03:46:18,546] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:46:20,960] INFO : Step 27000/500000 rank0 | Train Loss: 0.4126 | Train Loss Unweighted: 0.0427 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0189 | Regression_cos_phi: 0.0000 | Regression_energy: 0.0118 | Regression_eta: 0.0001 | Regression_pt: 0.0118 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.0189 | Regression_cos_phi: 0.0126 | Regression_energy: 0.1300 | Regression_eta: 0.0313 | Regression_pt: 0.1342 | Regression_sin_phi: 0.0855 | LR: 5.74e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8201s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0023 | max_backward: 0.0259 | max_optimizer_step: 0.0853 | max_tensorboard_logging: 0.2669 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3385.1 | uss_MB: 2189.6 | child_rss_MB: 23033.3 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:46:20,962] INFO : Running validation on rank0 +rank0 [2026-11-15 03:46:20,962] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:46:20,962] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:46:21,060] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 03:46:21,152] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:46:21,223] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:46:21,273] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:46:21,320] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:46:21,367] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:46:21,414] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:46:21,462] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:46:21,541] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:46:21,613] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:46:21,684] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:46:21,733] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:46:21,783] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:46:21,831] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:46:21,881] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:46:21,929] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:46:21,996] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:46:22,000] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:46:22,002] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:46:22,003] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:46:22,005] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:46:22,007] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:46:22,008] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:46:22,009] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:46:22,011] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:46:22,012] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:46:22,012] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:46:22,014] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:46:22,015] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:46:22,016] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:46:22,017] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:46:22,019] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:46:22,927] INFO : VALIDATION | Step=27000/500000 | Train Loss=0.4126 | Valid Loss=9.6160 | Train Loss Unweighted=0.0427 | Valid Loss Unweighted=2.1721 | Stale=25 | ETA=633.5m +rank0 [2026-11-15 03:46:22,928] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:46:23,054] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:46:23,070] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:46:23,122] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:46:23,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:46:23,794] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:46:23,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:46:24,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:46:24,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:46:24,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:46:24,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:46:24,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:46:24,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:46:25,098] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:46:25,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:46:25,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:46:25,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:46:25,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:46:25,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:46:26,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:46:26,120] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:46:26,128] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:46:26,518] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_27000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:46:26,846] INFO : Plotted number of elements +rank0 [2026-11-15 03:46:27,925] INFO : Plotted jets +rank0 [2026-11-15 03:46:29,476] INFO : Plotted jet ratio +rank0 [2026-11-15 03:47:07,847] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 03:47:07,847] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:47:41,275] INFO : Step 28000/500000 rank0 | Train Loss: 0.8055 | Train Loss Unweighted: 0.2805 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.2451 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0177 | Regression_eta: 0.0001 | Regression_pt: 0.0174 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0010 | Classification_binary: 0.2451 | Regression_cos_phi: 0.0490 | Regression_energy: 0.1953 | Regression_eta: 0.0439 | Regression_pt: 0.1973 | Regression_sin_phi: 0.0739 | LR: 6.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8295s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0248 | max_loss: 0.0024 | max_backward: 0.0263 | max_optimizer_step: 0.0912 | max_tensorboard_logging: 0.2581 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3414.1 | uss_MB: 2218.5 | child_rss_MB: 23033.9 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:47:41,277] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 03:47:41,328] INFO : Running validation on rank0 +rank0 [2026-11-15 03:47:41,328] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:47:41,328] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:47:41,409] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 03:47:41,486] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:47:41,556] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:47:41,607] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:47:41,654] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:47:41,699] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:47:41,746] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:47:41,793] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:47:41,870] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:47:41,940] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:47:42,010] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:47:42,056] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:47:42,102] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:47:42,148] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:47:42,194] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:47:42,244] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:47:42,310] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:47:42,314] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:47:42,316] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:47:42,317] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:47:42,319] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:47:42,320] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:47:42,322] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:47:42,322] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:47:42,324] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:47:42,325] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:47:42,326] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:47:42,327] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:47:42,329] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:47:42,330] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:47:42,332] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:47:42,333] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:47:43,232] INFO : VALIDATION | Step=28000/500000 | Train Loss=0.8055 | Valid Loss=9.3077 | Train Loss Unweighted=0.2805 | Valid Loss Unweighted=2.1454 | Stale=26 | ETA=632.2m +rank0 [2026-11-15 03:47:43,232] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:47:43,347] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:47:43,365] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:47:43,416] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:47:43,901] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:47:44,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:47:44,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:47:44,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:47:44,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:47:44,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:47:44,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:47:44,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:47:45,140] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:47:45,307] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:47:45,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:47:45,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:47:45,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:47:45,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:47:46,096] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:47:46,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:47:46,321] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:47:46,329] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:47:46,691] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_28000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:47:46,953] INFO : Plotted number of elements +rank0 [2026-11-15 03:47:47,822] INFO : Plotted jets +rank0 [2026-11-15 03:47:49,004] INFO : Plotted jet ratio +rank0 [2026-11-15 03:47:56,421] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 03:47:56,421] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:48:36,310] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 03:48:36,310] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:49:01,185] INFO : Step 29000/500000 rank0 | Train Loss: 0.6816 | Train Loss Unweighted: 0.0973 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.0686 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0142 | Regression_eta: 0.0002 | Regression_pt: 0.0135 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.0686 | Regression_cos_phi: 0.0398 | Regression_energy: 0.1568 | Regression_eta: 0.1105 | Regression_pt: 0.1539 | Regression_sin_phi: 0.1141 | LR: 6.29e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8509s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0022 | max_backward: 0.0268 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2981 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3435.8 | uss_MB: 2240.3 | child_rss_MB: 23033.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:49:01,187] INFO : Running validation on rank0 +rank0 [2026-11-15 03:49:01,187] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:49:01,187] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:49:01,270] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 03:49:01,343] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:49:01,414] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:49:01,463] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:49:01,510] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:49:01,555] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:49:01,601] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:49:01,648] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:49:01,726] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:49:01,798] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:49:01,869] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:49:01,916] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:49:01,961] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:49:02,008] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:49:02,054] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:49:02,100] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:49:02,165] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:49:02,169] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:49:02,171] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:49:02,172] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:49:02,174] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:49:02,175] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:49:02,177] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:49:02,178] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:49:02,179] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:49:02,180] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:49:02,181] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:49:02,182] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:49:02,184] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:49:02,185] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:49:02,187] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:49:02,188] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:49:03,090] INFO : VALIDATION | Step=29000/500000 | Train Loss=0.6816 | Valid Loss=9.8536 | Train Loss Unweighted=0.0973 | Valid Loss Unweighted=2.2811 | Stale=27 | ETA=630.7m +rank0 [2026-11-15 03:49:03,091] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:49:03,224] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:49:03,241] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:49:03,295] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:49:03,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:49:03,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:49:04,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:49:04,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:49:04,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:49:04,611] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:49:04,792] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:49:04,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:49:05,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:49:05,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:49:05,479] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:49:05,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:49:05,817] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:49:05,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:49:06,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:49:06,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:49:06,390] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:49:06,398] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:49:06,803] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_29000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:49:07,086] INFO : Plotted number of elements +rank0 [2026-11-15 03:49:08,109] INFO : Plotted jets +rank0 [2026-11-15 03:49:09,652] INFO : Plotted jet ratio +rank0 [2026-11-15 03:49:26,364] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 03:49:26,364] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:50:06,532] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 03:50:06,532] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:50:22,769] INFO : Step 30000/500000 rank0 | Train Loss: 1.0952 | Train Loss Unweighted: 0.2885 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.2559 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0155 | Regression_eta: 0.0004 | Regression_pt: 0.0162 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0018 | Classification_binary: 0.2559 | Regression_cos_phi: 0.0947 | Regression_energy: 0.1708 | Regression_eta: 0.1699 | Regression_pt: 0.1847 | Regression_sin_phi: 0.2174 | LR: 6.58e-05 | DataLoad Time: 0.0013s | Model Forward Time: 0.9805s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0004 | max_forward: 0.0355 | max_loss: 0.0031 | max_backward: 0.0275 | max_optimizer_step: 0.0976 | max_tensorboard_logging: 0.2963 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3455.0 | uss_MB: 2259.5 | child_rss_MB: 23033.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:50:22,771] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 03:50:22,824] INFO : Running validation on rank0 +rank0 [2026-11-15 03:50:22,824] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:50:22,824] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:50:22,906] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 03:50:22,986] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:50:23,059] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:50:23,110] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:50:23,160] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:50:23,209] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:50:23,258] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:50:23,307] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:50:23,387] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:50:23,461] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:50:23,533] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:50:23,581] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:50:23,629] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:50:23,677] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:50:23,725] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:50:23,774] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:50:23,841] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:50:23,845] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:50:23,846] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:50:23,848] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:50:23,850] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:50:23,851] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:50:23,853] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:50:23,853] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:50:23,854] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:50:23,856] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:50:23,856] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:50:23,858] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:50:23,859] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:50:23,860] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:50:23,861] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:50:23,863] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:50:24,893] INFO : VALIDATION | Step=30000/500000 | Train Loss=1.0952 | Valid Loss=9.9910 | Train Loss Unweighted=0.2885 | Valid Loss Unweighted=2.2803 | Stale=28 | ETA=629.7m +rank0 [2026-11-15 03:50:24,893] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:50:25,029] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:50:25,047] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:50:25,104] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:50:25,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:50:25,783] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:50:25,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:50:26,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:50:26,247] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:50:26,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:50:26,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:50:26,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:50:26,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:50:27,011] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:50:27,178] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:50:27,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:50:27,496] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:50:27,661] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:50:27,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:50:27,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:50:28,020] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:50:28,028] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:50:28,386] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:50:28,651] INFO : Plotted number of elements +rank0 [2026-11-15 03:50:29,520] INFO : Plotted jets +rank0 [2026-11-15 03:50:30,842] INFO : Plotted jet ratio +rank0 [2026-11-15 03:50:57,797] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 03:50:57,797] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:51:40,419] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 03:51:40,419] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:51:47,953] INFO : Step 31000/500000 rank0 | Train Loss: 0.7322 | Train Loss Unweighted: 0.0720 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.0376 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0171 | Regression_eta: 0.0003 | Regression_pt: 0.0167 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0051 | Classification_binary: 0.0376 | Regression_cos_phi: 0.0381 | Regression_energy: 0.1884 | Regression_eta: 0.1360 | Regression_pt: 0.1897 | Regression_sin_phi: 0.1373 | LR: 6.88e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8239s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0259 | max_optimizer_step: 0.0844 | max_tensorboard_logging: 0.2741 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3408.7 | uss_MB: 2213.2 | child_rss_MB: 23032.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:51:47,955] INFO : Running validation on rank0 +rank0 [2026-11-15 03:51:47,955] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:51:47,955] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:51:48,039] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 03:51:48,117] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:51:48,188] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:51:48,242] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:51:48,289] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:51:48,337] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:51:48,386] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:51:48,435] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:51:48,518] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:51:48,595] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:51:48,671] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:51:48,721] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:51:48,773] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:51:48,824] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:51:48,878] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:51:48,929] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:51:49,001] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:51:49,005] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:51:49,007] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:51:49,009] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:51:49,011] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:51:49,013] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:51:49,015] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:51:49,018] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:51:49,019] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:51:49,021] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:51:49,022] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:51:49,023] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:51:49,026] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:51:49,027] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:51:49,029] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:51:49,031] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:51:50,068] INFO : VALIDATION | Step=31000/500000 | Train Loss=0.7322 | Valid Loss=10.5751 | Train Loss Unweighted=0.0720 | Valid Loss Unweighted=2.4504 | Stale=29 | ETA=629.6m +rank0 [2026-11-15 03:51:50,068] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:51:50,206] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:51:50,222] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:51:50,276] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:51:50,770] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:51:50,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:51:51,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:51:51,235] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:51:51,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:51:51,541] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:51:51,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:51:51,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:51:52,002] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:51:52,156] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:51:52,310] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:51:52,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:51:52,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:51:52,768] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:51:52,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:51:53,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:51:53,153] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:51:53,160] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:51:53,523] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_31000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:51:53,784] INFO : Plotted number of elements +rank0 [2026-11-15 03:51:54,657] INFO : Plotted jets +rank0 [2026-11-15 03:51:55,932] INFO : Plotted jet ratio +rank0 [2026-11-15 03:52:29,639] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-15 03:52:29,640] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:53:07,425] INFO : Step 32000/500000 rank0 | Train Loss: 0.5597 | Train Loss Unweighted: 0.0735 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0484 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0122 | Regression_eta: 0.0001 | Regression_pt: 0.0124 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 85.1353 | Classification_binary: 1.0000 | Regression_cos_phi: 389.4189 | Regression_energy: 11.0202 | Regression_eta: 468.5324 | Regression_pt: 11.3655 | Regression_sin_phi: 1000.0000 | Task Weighted Losses: Classification: 0.0008 | Classification_binary: 0.0484 | Regression_cos_phi: 0.0650 | Regression_energy: 0.1342 | Regression_eta: 0.0702 | Regression_pt: 0.1412 | Regression_sin_phi: 0.0997 | LR: 7.19e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8462s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0025 | max_backward: 0.0262 | max_optimizer_step: 0.0909 | max_tensorboard_logging: 0.2695 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3520.3 | uss_MB: 2324.8 | child_rss_MB: 23030.5 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:53:07,427] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/checkpoints/checkpoint-32000.pth +rank0 [2026-11-15 03:53:07,477] INFO : Running validation on rank0 +rank0 [2026-11-15 03:53:07,477] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:53:07,477] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:53:07,558] INFO : saved batch0_epoch32000.parquet +rank0 [2026-11-15 03:53:07,628] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:53:07,699] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:53:07,750] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:53:07,797] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:53:07,843] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:53:07,889] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:53:07,936] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:53:08,014] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:53:08,084] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:53:08,153] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:53:08,199] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:53:08,245] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:53:08,291] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:53:08,337] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:53:08,383] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:53:08,448] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:53:08,452] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:53:08,454] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:53:08,455] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:53:08,457] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:53:08,458] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:53:08,460] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:53:08,461] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:53:08,463] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:53:08,464] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:53:08,465] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:53:08,466] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:53:08,468] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:53:08,468] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:53:08,470] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:53:08,471] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:53:09,371] INFO : VALIDATION | Step=32000/500000 | Train Loss=0.5597 | Valid Loss=11.0309 | Train Loss Unweighted=0.0735 | Valid Loss Unweighted=2.7526 | Stale=30 | ETA=628.0m +rank0 [2026-11-15 03:53:09,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:53:09,487] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:53:09,505] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:53:09,558] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:53:10,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:53:10,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:53:10,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:53:10,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:53:10,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:53:10,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:53:10,988] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:53:11,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:53:11,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:53:11,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:53:11,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:53:11,792] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:53:11,955] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:53:12,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:53:12,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:53:12,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:53:12,510] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 03:53:12,518] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:53:12,919] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_3mm_seed12345_20261115_030928_848509/preds_step_32000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:53:13,206] INFO : Plotted number of elements +rank0 [2026-11-15 03:53:14,210] INFO : Plotted jets +rank0 [2026-11-15 03:53:15,669] INFO : Plotted jet ratio +rank0 [2026-11-15 03:53:16,152] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 03:53:16,152] INFO : Training completed. Total time on device 0: 43.050min diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..9dcaf39fb5603cc1caf5a029276ad1c370a74b03 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa88e4f789d9f013d39f059af759e3fc6896ba338d6de67165c86c2b88980762 +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-38000.pth b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-38000.pth new file mode 100644 index 0000000000000000000000000000000000000000..4280af022a1284822d681fde5a8930a6f87aa5cb --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-38000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:775b95643f4b1511c4f14f928a41b6527b053452fd5aaf24f1362069092a5738 +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_1000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_1000.json new file mode 100644 index 0000000000000000000000000000000000000000..0ef34f14618a419209a49375dc03fee67ef4e443 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_1000.json @@ -0,0 +1 @@ +{"epoch": 1.7793594306049823, "train": {"Classification_binary": 0.8708857297897339, "Classification": 0.010902533307671547, "Regression_pt": 0.09629693627357483, "Regression_eta": 0.0003242881502956152, "Regression_sin_phi": 0.00015078989963512868, "Regression_cos_phi": 0.0002937149256467819, "Regression_energy": 0.09692108631134033, "Total": 4.538215160369873, "Total_unweighted": 1.0757750786578981}, "train_group": {}, "valid": {"Classification_binary": 1.0429651737213135, "Classification": 0.010072633624076843, "Regression_pt": 0.10195917636156082, "Regression_eta": 0.013110966421663761, "Regression_sin_phi": 0.0025438396260142326, "Regression_cos_phi": 0.003033066401258111, "Regression_energy": 0.10982118546962738, "Total": 8.965044975280762, "Total_unweighted": 1.2835060416255146, "metrics/particle/class_1/efficiency": 0.9675173759460449, "metrics/particle/class_1/pid_accuracy": 0.9784172773361206, "metrics/particle/class_2/efficiency": 0.3913043439388275, "metrics/particle/class_2/pid_accuracy": 0.7777777910232544, "metrics/particle/class_3/efficiency": 0.8769841194152832, "metrics/particle/class_3/pid_accuracy": 0.9954751133918762, "metrics/particle/class_4/efficiency": 0.9509803652763367, "metrics/particle/class_4/pid_accuracy": 0.9484536051750183, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.06800000369548798, "metrics/particle/count/mae": 0.2199999988079071, "metrics/particle/count/prediction_mean": 4.547999858856201, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.038292545825242996, "metrics/particle/event/energy_response_mean": 0.9879692792892456, "metrics/particle/event/met_abs_error": 1.0551831722259521, "metrics/particle/event/scalar_pt_relative_abs_error": 0.036701761186122894, "metrics/particle/event/scalar_pt_response_mean": 0.9918332099914551, "metrics/particle/event/vector_pt_closure": 0.041084133088588715, "metrics/particle/matched/delta_eta_abs_mean": 0.00800997857004404, "metrics/particle/matched/delta_phi_abs_mean": 0.007854638621211052, "metrics/particle/matched/delta_r_mean": 0.012520255520939827, "metrics/particle/matched/energy_relative_abs_mean": 0.060506582260131836, "metrics/particle/matched/pid_accuracy": 0.980169951915741, "metrics/particle/matched/pt_relative_abs_mean": 0.06167283281683922, "metrics/particle/matching/duplicate_fraction": 0.01231310423463583, "metrics/particle/matching/efficiency": 0.917677640914917, "metrics/particle/matching/f1": 0.9244871139526367, "metrics/particle/matching/purity": 0.9313983917236328, "metrics/particle/matching/target_coverage_dr0p05": 0.9081456065177917, "metrics/particle/matching/target_coverage_dr0p10": 0.9384748935699463, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8613518476486206, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9350086450576782}, "step_train_time": 0.4743351936340332, "step_valid_time": 2.603071694999926, "step_total_time": 112.51167106628418, "elapsed_training_loop_seconds": 112.51167106628418, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0000730702437506, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02684703127458188, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.009280742459396751, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.360836601000301, "step_plot_time": 3.8678599310005666, "step_validation_cycle_time": 11.241433066000354} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..7793bf66f79f15bd147fe1e597b6340cd2d71276 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 0.4373061954975128, "Classification": 0.002260626293718815, "Regression_pt": 0.014984077773988247, "Regression_eta": 0.0006080338498577476, "Regression_sin_phi": 0.00015203763905446976, "Regression_cos_phi": 0.00014053504855837673, "Regression_energy": 0.014437653124332428, "Total": 1.2373429536819458, "Total_unweighted": 0.4698891592270229}, "train_group": {}, "valid": {"Classification_binary": 1.0462095737457275, "Classification": 0.007677855901420116, "Regression_pt": 0.08728783577680588, "Regression_eta": 0.01347900927066803, "Regression_sin_phi": 0.0026529505848884583, "Regression_cos_phi": 0.0022969930432736874, "Regression_energy": 0.09399638324975967, "Total": 8.262327194213867, "Total_unweighted": 1.2536006015725434, "metrics/particle/class_1/efficiency": 0.9698376059532166, "metrics/particle/class_1/pid_accuracy": 0.9856459498405457, "metrics/particle/class_2/efficiency": 0.47826087474823, "metrics/particle/class_2/pid_accuracy": 0.8181818127632141, "metrics/particle/class_3/efficiency": 0.89682537317276, "metrics/particle/class_3/pid_accuracy": 0.991150438785553, "metrics/particle/class_4/efficiency": 0.9509803652763367, "metrics/particle/class_4/pid_accuracy": 0.969072163105011, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.03200000151991844, "metrics/particle/count/mae": 0.23199999332427979, "metrics/particle/count/prediction_mean": 4.584000110626221, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.03799224644899368, "metrics/particle/event/energy_response_mean": 0.9918907880783081, "metrics/particle/event/met_abs_error": 1.041551947593689, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03840558975934982, "metrics/particle/event/scalar_pt_response_mean": 0.9908778071403503, "metrics/particle/event/vector_pt_closure": 0.03991645947098732, "metrics/particle/matched/delta_eta_abs_mean": 0.00759156234562397, "metrics/particle/matched/delta_phi_abs_mean": 0.0065214685164391994, "metrics/particle/matched/delta_r_mean": 0.011133338324725628, "metrics/particle/matched/energy_relative_abs_mean": 0.05806689336895943, "metrics/particle/matched/pid_accuracy": 0.983208954334259, "metrics/particle/matched/pt_relative_abs_mean": 0.058722928166389465, "metrics/particle/matching/duplicate_fraction": 0.013961605727672577, "metrics/particle/matching/efficiency": 0.9289427995681763, "metrics/particle/matching/f1": 0.9321739077568054, "metrics/particle/matching/purity": 0.9354275465011597, "metrics/particle/matching/target_coverage_dr0p05": 0.915944516658783, "metrics/particle/matching/target_coverage_dr0p10": 0.9480069279670715, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8700173497200012, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9419410824775696}, "step_train_time": 0.47293591499328613, "step_valid_time": 2.8399366209996515, "step_total_time": 1178.1237280368805, "elapsed_training_loop_seconds": 1178.1237280368805, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9991576491921824, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02486526776634501, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9930232558139535, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.991869918699187, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0069767441860465115, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9872241579558653, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.216954105999321, "step_plot_time": 3.582693839000058, "step_validation_cycle_time": 10.959000677000404} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_11000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_11000.json new file mode 100644 index 0000000000000000000000000000000000000000..86a8ba98b63d583b05e31f6f31e240c1a4fdbccc --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_11000.json @@ -0,0 +1 @@ +{"epoch": 19.572953736654803, "train": {"Classification_binary": 0.47749531269073486, "Classification": 0.0005897263763472438, "Regression_pt": 0.03540519252419472, "Regression_eta": 0.00026466400595381856, "Regression_sin_phi": 0.0002516723470762372, "Regression_cos_phi": 0.0003597419708967209, "Regression_energy": 0.035173699259757996, "Total": 1.7160429954528809, "Total_unweighted": 0.5495400091749616}, "train_group": {}, "valid": {"Classification_binary": 1.075204849243164, "Classification": 0.008795577101409435, "Regression_pt": 0.09127817302942276, "Regression_eta": 0.014001836068928242, "Regression_sin_phi": 0.00277521344833076, "Regression_cos_phi": 0.0024274291936308146, "Regression_energy": 0.09791741520166397, "Total": 8.681394577026367, "Total_unweighted": 1.29240049328655, "metrics/particle/class_1/efficiency": 0.9698376059532166, "metrics/particle/class_1/pid_accuracy": 0.9880383014678955, "metrics/particle/class_2/efficiency": 0.47826087474823, "metrics/particle/class_2/pid_accuracy": 0.8181818127632141, "metrics/particle/class_3/efficiency": 0.886904776096344, "metrics/particle/class_3/pid_accuracy": 0.9977628588676453, "metrics/particle/class_4/efficiency": 0.970588207244873, "metrics/particle/class_4/pid_accuracy": 0.9696969985961914, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.957446813583374, "metrics/particle/count/bias_mean": -0.03200000151991844, "metrics/particle/count/mae": 0.23999999463558197, "metrics/particle/count/prediction_mean": 4.584000110626221, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.0403439886868, "metrics/particle/event/energy_response_mean": 0.9925850629806519, "metrics/particle/event/met_abs_error": 1.0589544773101807, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04009684920310974, "metrics/particle/event/scalar_pt_response_mean": 0.9924674034118652, "metrics/particle/event/vector_pt_closure": 0.040413349866867065, "metrics/particle/matched/delta_eta_abs_mean": 0.007398724090307951, "metrics/particle/matched/delta_phi_abs_mean": 0.006404669024050236, "metrics/particle/matched/delta_r_mean": 0.010905657894909382, "metrics/particle/matched/energy_relative_abs_mean": 0.05774044245481491, "metrics/particle/matched/pid_accuracy": 0.9859681725502014, "metrics/particle/matched/pt_relative_abs_mean": 0.05877574905753136, "metrics/particle/matching/duplicate_fraction": 0.013089004904031754, "metrics/particle/matching/efficiency": 0.9263431429862976, "metrics/particle/matching/f1": 0.9295651912689209, "metrics/particle/matching/purity": 0.9328097701072693, "metrics/particle/matching/target_coverage_dr0p05": 0.9124783277511597, "metrics/particle/matching/target_coverage_dr0p10": 0.9454072713851929, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8648180365562439, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9393414258956909}, "step_train_time": 0.46468305587768555, "step_valid_time": 2.8417942040005073, "step_total_time": 1297.1582701206207, "elapsed_training_loop_seconds": 1297.1582701206207, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.998885190368362, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.024292008464160686, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9895470383275261, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.009302325581395349, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9895470383275261, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.202836877999289, "step_plot_time": 3.5325073720005093, "step_validation_cycle_time": 10.902973246000329} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_12000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_12000.json new file mode 100644 index 0000000000000000000000000000000000000000..fd93414ff7daf5c0e9b972d457cd5c764ac18a2f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_12000.json @@ -0,0 +1 @@ +{"epoch": 21.352313167259787, "train": {"Classification_binary": 1.0508439540863037, "Classification": 7.702039874857292e-05, "Regression_pt": 0.020268088206648827, "Regression_eta": 0.0001810449903132394, "Regression_sin_phi": 0.00029087759321555495, "Regression_cos_phi": 0.0002068822068395093, "Regression_energy": 0.01841280609369278, "Total": 1.8045876026153564, "Total_unweighted": 1.0902806735757622}, "train_group": {}, "valid": {"Classification_binary": 1.1226933002471924, "Classification": 0.00893421284854412, "Regression_pt": 0.08006793260574341, "Regression_eta": 0.013919172808527946, "Regression_sin_phi": 0.002717331051826477, "Regression_cos_phi": 0.0023980119731277227, "Regression_energy": 0.08702493458986282, "Total": 8.407660484313965, "Total_unweighted": 1.3177548961248249, "metrics/particle/class_1/efficiency": 0.9698376059532166, "metrics/particle/class_1/pid_accuracy": 0.9904305934906006, "metrics/particle/class_2/efficiency": 0.43478259444236755, "metrics/particle/class_2/pid_accuracy": 0.800000011920929, "metrics/particle/class_3/efficiency": 0.8928571343421936, "metrics/particle/class_3/pid_accuracy": 0.9911110997200012, "metrics/particle/class_4/efficiency": 0.9803921580314636, "metrics/particle/class_4/pid_accuracy": 0.9700000286102295, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.04399999976158142, "metrics/particle/count/mae": 0.21199999749660492, "metrics/particle/count/prediction_mean": 4.572000026702881, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.043986402451992035, "metrics/particle/event/energy_response_mean": 0.9910827279090881, "metrics/particle/event/met_abs_error": 1.126129388809204, "metrics/particle/event/scalar_pt_relative_abs_error": 0.043196901679039, "metrics/particle/event/scalar_pt_response_mean": 0.9908735752105713, "metrics/particle/event/vector_pt_closure": 0.043004050850868225, "metrics/particle/matched/delta_eta_abs_mean": 0.00794279295951128, "metrics/particle/matched/delta_phi_abs_mean": 0.006443156860768795, "metrics/particle/matched/delta_r_mean": 0.011430074460804462, "metrics/particle/matched/energy_relative_abs_mean": 0.058046016842126846, "metrics/particle/matched/pid_accuracy": 0.9850746393203735, "metrics/particle/matched/pt_relative_abs_mean": 0.05844999849796295, "metrics/particle/matching/duplicate_fraction": 0.010498687624931335, "metrics/particle/matching/efficiency": 0.9289427995681763, "metrics/particle/matching/f1": 0.9333913922309875, "metrics/particle/matching/purity": 0.9378827810287476, "metrics/particle/matching/target_coverage_dr0p05": 0.9150779843330383, "metrics/particle/matching/target_coverage_dr0p10": 0.9471403956413269, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8656845688819885, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.941074550151825}, "step_train_time": 0.46946167945861816, "step_valid_time": 2.815959548000137, "step_total_time": 1416.2583837509155, "elapsed_training_loop_seconds": 1416.2583837509155, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9995910624238715, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.022252921470016784, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9953379953379954, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9930232558139535, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.004662004662004662, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.191122504999839, "step_plot_time": 3.573118983000313, "step_validation_cycle_time": 10.911522281000543} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_13000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_13000.json new file mode 100644 index 0000000000000000000000000000000000000000..42438bb707beceb76ff2311f8cc1978f2e1e6e09 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_13000.json @@ -0,0 +1 @@ +{"epoch": 23.131672597864767, "train": {"Classification_binary": 0.4390732944011688, "Classification": 0.00020273895643185824, "Regression_pt": 0.021760206669569016, "Regression_eta": 0.00014491609181277454, "Regression_sin_phi": 0.0001484240056015551, "Regression_cos_phi": 0.00011293797433609143, "Regression_energy": 0.02202138490974903, "Total": 1.1522516012191772, "Total_unweighted": 0.48346390300866915}, "train_group": {}, "valid": {"Classification_binary": 1.1192060708999634, "Classification": 0.009589718654751778, "Regression_pt": 0.08596181124448776, "Regression_eta": 0.01389970351010561, "Regression_sin_phi": 0.0028834796976298094, "Regression_cos_phi": 0.0025347708724439144, "Regression_energy": 0.0929465964436531, "Total": 8.748284339904785, "Total_unweighted": 1.3270221513230354, "metrics/particle/class_1/efficiency": 0.9744779467582703, "metrics/particle/class_1/pid_accuracy": 0.9857142567634583, "metrics/particle/class_2/efficiency": 0.43478259444236755, "metrics/particle/class_2/pid_accuracy": 0.800000011920929, "metrics/particle/class_3/efficiency": 0.886904776096344, "metrics/particle/class_3/pid_accuracy": 0.9955257177352905, "metrics/particle/class_4/efficiency": 0.970588207244873, "metrics/particle/class_4/pid_accuracy": 0.9696969985961914, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.05999999865889549, "metrics/particle/count/mae": 0.2280000001192093, "metrics/particle/count/prediction_mean": 4.556000232696533, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.039480727165937424, "metrics/particle/event/energy_response_mean": 0.990302324295044, "metrics/particle/event/met_abs_error": 1.1231404542922974, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03968570753931999, "metrics/particle/event/scalar_pt_response_mean": 0.9880926012992859, "metrics/particle/event/vector_pt_closure": 0.04228261113166809, "metrics/particle/matched/delta_eta_abs_mean": 0.007355404552072287, "metrics/particle/matched/delta_phi_abs_mean": 0.006739255506545305, "metrics/particle/matched/delta_r_mean": 0.011104862205684185, "metrics/particle/matched/energy_relative_abs_mean": 0.058608416467905045, "metrics/particle/matched/pid_accuracy": 0.9850467443466187, "metrics/particle/matched/pt_relative_abs_mean": 0.058806587010622025, "metrics/particle/matching/duplicate_fraction": 0.012291483581066132, "metrics/particle/matching/efficiency": 0.927209734916687, "metrics/particle/matching/f1": 0.9332751631736755, "metrics/particle/matching/purity": 0.9394205212593079, "metrics/particle/matching/target_coverage_dr0p05": 0.9124783277511597, "metrics/particle/matching/target_coverage_dr0p10": 0.9436741471290588, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8708838820457458, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9428076148033142}, "step_train_time": 0.49733734130859375, "step_valid_time": 2.8517229290000614, "step_total_time": 1535.3018231391907, "elapsed_training_loop_seconds": 1535.3018231391907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9996958427839555, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02374994119963436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.006993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9883449883449883, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.986046511627907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.16754561200014, "step_plot_time": 3.5412552509997113, "step_validation_cycle_time": 10.880834749999849} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_14000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_14000.json new file mode 100644 index 0000000000000000000000000000000000000000..08c09c74f032e79654936bb8518a57d3a1bedf2e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_14000.json @@ -0,0 +1 @@ +{"epoch": 24.91103202846975, "train": {"Classification_binary": 0.3974391222000122, "Classification": 0.00020026543643325567, "Regression_pt": 0.053817927837371826, "Regression_eta": 0.0002822999667841941, "Regression_sin_phi": 0.0002397749194642529, "Regression_cos_phi": 0.00010985440167132765, "Regression_energy": 0.056035567075014114, "Total": 2.0174436569213867, "Total_unweighted": 0.5081248118367512}, "train_group": {}, "valid": {"Classification_binary": 1.1691009998321533, "Classification": 0.00999525561928749, "Regression_pt": 0.0892263799905777, "Regression_eta": 0.014877790585160255, "Regression_sin_phi": 0.002814296167343855, "Regression_cos_phi": 0.0025552192237228155, "Regression_energy": 0.09613416343927383, "Total": 9.027403831481934, "Total_unweighted": 1.3847041048575193, "metrics/particle/class_1/efficiency": 0.9721577763557434, "metrics/particle/class_1/pid_accuracy": 0.9832935333251953, "metrics/particle/class_2/efficiency": 0.3913043439388275, "metrics/particle/class_2/pid_accuracy": 0.8888888955116272, "metrics/particle/class_3/efficiency": 0.886904776096344, "metrics/particle/class_3/pid_accuracy": 0.9932885766029358, "metrics/particle/class_4/efficiency": 0.9607843160629272, "metrics/particle/class_4/pid_accuracy": 0.9795918464660645, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.957446813583374, "metrics/particle/count/bias_mean": -0.04800000041723251, "metrics/particle/count/mae": 0.23999999463558197, "metrics/particle/count/prediction_mean": 4.567999839782715, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.04107557609677315, "metrics/particle/event/energy_response_mean": 0.9958195686340332, "metrics/particle/event/met_abs_error": 1.1419925689697266, "metrics/particle/event/scalar_pt_relative_abs_error": 0.037653788924217224, "metrics/particle/event/scalar_pt_response_mean": 0.994640588760376, "metrics/particle/event/vector_pt_closure": 0.04079221934080124, "metrics/particle/matched/delta_eta_abs_mean": 0.007492199540138245, "metrics/particle/matched/delta_phi_abs_mean": 0.007401963230222464, "metrics/particle/matched/delta_r_mean": 0.011697323061525822, "metrics/particle/matched/energy_relative_abs_mean": 0.057912759482860565, "metrics/particle/matched/pid_accuracy": 0.9840674996376038, "metrics/particle/matched/pt_relative_abs_mean": 0.05833017826080322, "metrics/particle/matching/duplicate_fraction": 0.010507880710065365, "metrics/particle/matching/efficiency": 0.9246100783348083, "metrics/particle/matching/f1": 0.9294425249099731, "metrics/particle/matching/purity": 0.9343257546424866, "metrics/particle/matching/target_coverage_dr0p05": 0.9124783277511597, "metrics/particle/matching/target_coverage_dr0p10": 0.9402079582214355, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8544194102287292, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9376083016395569}, "step_train_time": 0.49782705307006836, "step_valid_time": 2.857940253000379, "step_total_time": 1654.1244277954102, "elapsed_training_loop_seconds": 1654.1244277954102, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0014241061502696, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02374469670600754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9930232558139535, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.991869918699187, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0069767441860465115, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9895470383275261, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.232103215999814, "step_plot_time": 3.529899226999987, "step_validation_cycle_time": 10.937629078999635} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_15000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_15000.json new file mode 100644 index 0000000000000000000000000000000000000000..bdbb12f6782185a0a6f04390f289539935f317da --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_15000.json @@ -0,0 +1 @@ +{"epoch": 26.690391459074732, "train": {"Classification_binary": 0.7016552686691284, "Classification": 0.002545752562582493, "Regression_pt": 0.03194578364491463, "Regression_eta": 0.0004339914012234658, "Regression_sin_phi": 0.00029363803332671523, "Regression_cos_phi": 0.0005973537918180227, "Regression_energy": 0.033488426357507706, "Total": 2.1749179363250732, "Total_unweighted": 0.7709602144605014}, "train_group": {}, "valid": {"Classification_binary": 1.200576663017273, "Classification": 0.009555653668940067, "Regression_pt": 0.08761466294527054, "Regression_eta": 0.014133099466562271, "Regression_sin_phi": 0.0029531975742429495, "Regression_cos_phi": 0.002585451351478696, "Regression_energy": 0.09456995129585266, "Total": 8.958086967468262, "Total_unweighted": 1.4119886793196201, "metrics/particle/class_1/efficiency": 0.9721577763557434, "metrics/particle/class_1/pid_accuracy": 0.9832935333251953, "metrics/particle/class_2/efficiency": 0.43478259444236755, "metrics/particle/class_2/pid_accuracy": 0.8999999761581421, "metrics/particle/class_3/efficiency": 0.8888888955116272, "metrics/particle/class_3/pid_accuracy": 0.9933035969734192, "metrics/particle/class_4/efficiency": 0.9803921580314636, "metrics/particle/class_4/pid_accuracy": 0.9800000190734863, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.03200000151991844, "metrics/particle/count/mae": 0.23199999332427979, "metrics/particle/count/prediction_mean": 4.584000110626221, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.04177817329764366, "metrics/particle/event/energy_response_mean": 0.9955607056617737, "metrics/particle/event/met_abs_error": 1.0825029611587524, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04050195962190628, "metrics/particle/event/scalar_pt_response_mean": 0.9952476620674133, "metrics/particle/event/vector_pt_closure": 0.04166196659207344, "metrics/particle/matched/delta_eta_abs_mean": 0.007418495137244463, "metrics/particle/matched/delta_phi_abs_mean": 0.007261407561600208, "metrics/particle/matched/delta_r_mean": 0.011571118608117104, "metrics/particle/matched/energy_relative_abs_mean": 0.05755273252725601, "metrics/particle/matched/pid_accuracy": 0.9850606918334961, "metrics/particle/matched/pt_relative_abs_mean": 0.057650890201330185, "metrics/particle/matching/duplicate_fraction": 0.010471204295754433, "metrics/particle/matching/efficiency": 0.9280762672424316, "metrics/particle/matching/f1": 0.9313043355941772, "metrics/particle/matching/purity": 0.9345549941062927, "metrics/particle/matching/target_coverage_dr0p05": 0.9142114520072937, "metrics/particle/matching/target_coverage_dr0p10": 0.9454072713851929, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8665511012077332, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9393414258956909}, "step_train_time": 0.46742868423461914, "step_valid_time": 2.8195066549997136, "step_total_time": 1772.836932182312, "elapsed_training_loop_seconds": 1772.836932182312, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9994896968940679, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02439927791449359, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9930232558139535, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.991869918699187, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0069767441860465115, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9872241579558653, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.224519673999566, "step_plot_time": 3.58092097700046, "step_validation_cycle_time": 10.949134088000392} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_16000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_16000.json new file mode 100644 index 0000000000000000000000000000000000000000..ed340b0283ad79db55dee00cb1cd3920e0baa714 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_16000.json @@ -0,0 +1 @@ +{"epoch": 28.469750889679716, "train": {"Classification_binary": 0.4323481321334839, "Classification": 0.00032700886367820203, "Regression_pt": 0.011780593544244766, "Regression_eta": 0.00012590031838044524, "Regression_sin_phi": 4.4781645556213334e-05, "Regression_cos_phi": 8.350996358785778e-05, "Regression_energy": 0.011842014268040657, "Total": 0.8303743600845337, "Total_unweighted": 0.45655194073697203}, "train_group": {}, "valid": {"Classification_binary": 1.177276372909546, "Classification": 0.011114070191979408, "Regression_pt": 0.08606385439634323, "Regression_eta": 0.014564297161996365, "Regression_sin_phi": 0.0029592392966151237, "Regression_cos_phi": 0.0027176777366548777, "Regression_energy": 0.0932173803448677, "Total": 9.148004531860352, "Total_unweighted": 1.3879128920380026, "metrics/particle/class_1/efficiency": 0.9651972055435181, "metrics/particle/class_1/pid_accuracy": 0.9831730723381042, "metrics/particle/class_2/efficiency": 0.3913043439388275, "metrics/particle/class_2/pid_accuracy": 0.8888888955116272, "metrics/particle/class_3/efficiency": 0.89682537317276, "metrics/particle/class_3/pid_accuracy": 0.9933628439903259, "metrics/particle/class_4/efficiency": 0.970588207244873, "metrics/particle/class_4/pid_accuracy": 0.9696969985961914, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.012000000104308128, "metrics/particle/count/mae": 0.2280000001192093, "metrics/particle/count/prediction_mean": 4.604000091552734, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.0422029122710228, "metrics/particle/event/energy_response_mean": 0.9972337484359741, "metrics/particle/event/met_abs_error": 1.114978313446045, "metrics/particle/event/scalar_pt_relative_abs_error": 0.039742764085531235, "metrics/particle/event/scalar_pt_response_mean": 0.9963706731796265, "metrics/particle/event/vector_pt_closure": 0.04117710888385773, "metrics/particle/matched/delta_eta_abs_mean": 0.0073514473624527454, "metrics/particle/matched/delta_phi_abs_mean": 0.006505155935883522, "metrics/particle/matched/delta_r_mean": 0.01093064621090889, "metrics/particle/matched/energy_relative_abs_mean": 0.05691144987940788, "metrics/particle/matched/pid_accuracy": 0.9841121435165405, "metrics/particle/matched/pt_relative_abs_mean": 0.057396721094846725, "metrics/particle/matching/duplicate_fraction": 0.014769765548408031, "metrics/particle/matching/efficiency": 0.927209734916687, "metrics/particle/matching/f1": 0.9284164905548096, "metrics/particle/matching/purity": 0.9296264052391052, "metrics/particle/matching/target_coverage_dr0p05": 0.917677640914917, "metrics/particle/matching/target_coverage_dr0p10": 0.9454072713851929, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8622183799743652, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.941074550151825}, "step_train_time": 0.4835500717163086, "step_valid_time": 2.853537408000193, "step_total_time": 1892.5291860103607, "elapsed_training_loop_seconds": 1892.5291860103607, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9986861720361437, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02383333624393713, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.013921113689095127, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9860788863109049, "step_test_time": 4.162476993999917, "step_plot_time": 3.578056499000013, "step_validation_cycle_time": 10.917263970000022} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_17000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_17000.json new file mode 100644 index 0000000000000000000000000000000000000000..fdab2053590153522971c91b7fb4a300311fb9cb --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_17000.json @@ -0,0 +1 @@ +{"epoch": 30.249110320284696, "train": {"Classification_binary": 0.2857922315597534, "Classification": 0.00023997311654966325, "Regression_pt": 0.014343885704874992, "Regression_eta": 0.00019264704314991832, "Regression_sin_phi": 9.48041706578806e-05, "Regression_cos_phi": 0.00010605558782117441, "Regression_energy": 0.014461664482951164, "Total": 0.7869860529899597, "Total_unweighted": 0.3152312616657582}, "train_group": {}, "valid": {"Classification_binary": 1.2566972970962524, "Classification": 0.013247883878648281, "Regression_pt": 0.09196065366268158, "Regression_eta": 0.014757741242647171, "Regression_sin_phi": 0.0030380375683307648, "Regression_cos_phi": 0.002533070743083954, "Regression_energy": 0.09901653975248337, "Total": 9.591293334960938, "Total_unweighted": 1.4812512239441276, "metrics/particle/class_1/efficiency": 0.9767981171607971, "metrics/particle/class_1/pid_accuracy": 0.9881235361099243, "metrics/particle/class_2/efficiency": 0.3913043439388275, "metrics/particle/class_2/pid_accuracy": 0.8888888955116272, "metrics/particle/class_3/efficiency": 0.898809552192688, "metrics/particle/class_3/pid_accuracy": 0.9955849647521973, "metrics/particle/class_4/efficiency": 0.970588207244873, "metrics/particle/class_4/pid_accuracy": 0.9797979593276978, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.957446813583374, "metrics/particle/count/bias_mean": -0.02800000086426735, "metrics/particle/count/mae": 0.19599999487400055, "metrics/particle/count/prediction_mean": 4.5879998207092285, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.03892766684293747, "metrics/particle/event/energy_response_mean": 1.0002745389938354, "metrics/particle/event/met_abs_error": 1.0629212856292725, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03718242421746254, "metrics/particle/event/scalar_pt_response_mean": 1.000505805015564, "metrics/particle/event/vector_pt_closure": 0.039240699261426926, "metrics/particle/matched/delta_eta_abs_mean": 0.007570940535515547, "metrics/particle/matched/delta_phi_abs_mean": 0.007319053169339895, "metrics/particle/matched/delta_r_mean": 0.01177351176738739, "metrics/particle/matched/energy_relative_abs_mean": 0.059322062879800797, "metrics/particle/matched/pid_accuracy": 0.9869888424873352, "metrics/particle/matched/pt_relative_abs_mean": 0.05975490063428879, "metrics/particle/matching/duplicate_fraction": 0.01133391447365284, "metrics/particle/matching/efficiency": 0.9324089884757996, "metrics/particle/matching/f1": 0.9352455735206604, "metrics/particle/matching/purity": 0.9380993843078613, "metrics/particle/matching/target_coverage_dr0p05": 0.9185441732406616, "metrics/particle/matching/target_coverage_dr0p10": 0.9462738037109375, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8743500709533691, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9471403956413269}, "step_train_time": 0.4978344440460205, "step_valid_time": 2.8892744380000295, "step_total_time": 2012.0446774959564, "elapsed_training_loop_seconds": 2012.0446774959564, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000685960052847, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.023851329558305823, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.01160092807424594, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.217163846000403, "step_plot_time": 3.4914201359997605, "step_validation_cycle_time": 10.92252345199995} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_18000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_18000.json new file mode 100644 index 0000000000000000000000000000000000000000..bfdd9f4c693f4b4f0f78b2951e66a60d78e0e359 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_18000.json @@ -0,0 +1 @@ +{"epoch": 32.02846975088968, "train": {"Classification_binary": 0.1248665452003479, "Classification": 2.530678284529131e-05, "Regression_pt": 0.023661211133003235, "Regression_eta": 0.00018320756498724222, "Regression_sin_phi": 7.495361205656081e-05, "Regression_cos_phi": 0.00019611744210124016, "Regression_energy": 0.023375526070594788, "Total": 0.8504797220230103, "Total_unweighted": 0.17238286780593626}, "train_group": {}, "valid": {"Classification_binary": 1.2836099863052368, "Classification": 0.013139922171831131, "Regression_pt": 0.08411549031734467, "Regression_eta": 0.01465629879385233, "Regression_sin_phi": 0.002789707388728857, "Regression_cos_phi": 0.002689109416678548, "Regression_energy": 0.09054934978485107, "Total": 9.288917541503906, "Total_unweighted": 1.4915498641785234, "metrics/particle/class_1/efficiency": 0.9675173759460449, "metrics/particle/class_1/pid_accuracy": 0.9856114983558655, "metrics/particle/class_2/efficiency": 0.43478259444236755, "metrics/particle/class_2/pid_accuracy": 0.8999999761581421, "metrics/particle/class_3/efficiency": 0.89682537317276, "metrics/particle/class_3/pid_accuracy": 0.997787594795227, "metrics/particle/class_4/efficiency": 0.9607843160629272, "metrics/particle/class_4/pid_accuracy": 0.9795918464660645, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.04399999976158142, "metrics/particle/count/mae": 0.24400000274181366, "metrics/particle/count/prediction_mean": 4.572000026702881, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.04404878988862038, "metrics/particle/event/energy_response_mean": 0.9931826591491699, "metrics/particle/event/met_abs_error": 1.149718999862671, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04208574816584587, "metrics/particle/event/scalar_pt_response_mean": 0.9938487410545349, "metrics/particle/event/vector_pt_closure": 0.042620424181222916, "metrics/particle/matched/delta_eta_abs_mean": 0.007059672847390175, "metrics/particle/matched/delta_phi_abs_mean": 0.006335601210594177, "metrics/particle/matched/delta_r_mean": 0.010661070235073566, "metrics/particle/matched/energy_relative_abs_mean": 0.058530475944280624, "metrics/particle/matched/pid_accuracy": 0.9878618121147156, "metrics/particle/matched/pt_relative_abs_mean": 0.058623235672712326, "metrics/particle/matching/duplicate_fraction": 0.011373578570783138, "metrics/particle/matching/efficiency": 0.9280762672424316, "metrics/particle/matching/f1": 0.9325206875801086, "metrics/particle/matching/purity": 0.9370078444480896, "metrics/particle/matching/target_coverage_dr0p05": 0.9142114520072937, "metrics/particle/matching/target_coverage_dr0p10": 0.9445407390594482, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8648180365562439, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.941074550151825}, "step_train_time": 0.48351097106933594, "step_valid_time": 2.867486610000924, "step_total_time": 2131.9396641254425, "elapsed_training_loop_seconds": 2131.9396641254425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0020784947938797, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.023184930716790064, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.006993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.986013986013986, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9837209302325581, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.25499371099977, "step_plot_time": 3.581547825000598, "step_validation_cycle_time": 11.0422017460005} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_19000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_19000.json new file mode 100644 index 0000000000000000000000000000000000000000..2b8a7b798ba172b7c13fbee3948e383fe41cac10 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_19000.json @@ -0,0 +1 @@ +{"epoch": 33.80782918149466, "train": {"Classification_binary": 0.541103720664978, "Classification": 0.000497574859764427, "Regression_pt": 0.013999596238136292, "Regression_eta": 0.0002782578521873802, "Regression_sin_phi": 0.00038213832885958254, "Regression_cos_phi": 0.0004901770153082907, "Regression_energy": 0.013683894649147987, "Total": 1.3459721803665161, "Total_unweighted": 0.570435359608382}, "train_group": {}, "valid": {"Classification_binary": 1.3645679950714111, "Classification": 0.016816169023513794, "Regression_pt": 0.09440527111291885, "Regression_eta": 0.014782186597585678, "Regression_sin_phi": 0.003165866481140256, "Regression_cos_phi": 0.0025161937810480595, "Regression_energy": 0.1004163920879364, "Total": 10.14936351776123, "Total_unweighted": 1.5966700741555542, "metrics/particle/class_1/efficiency": 0.9721577763557434, "metrics/particle/class_1/pid_accuracy": 0.9832935333251953, "metrics/particle/class_2/efficiency": 0.47826087474823, "metrics/particle/class_2/pid_accuracy": 0.9090909361839294, "metrics/particle/class_3/efficiency": 0.8928571343421936, "metrics/particle/class_3/pid_accuracy": 0.9933333396911621, "metrics/particle/class_4/efficiency": 0.9607843160629272, "metrics/particle/class_4/pid_accuracy": 0.9795918464660645, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.035999998450279236, "metrics/particle/count/mae": 0.2280000001192093, "metrics/particle/count/prediction_mean": 4.579999923706055, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.04329242929816246, "metrics/particle/event/energy_response_mean": 0.9936046600341797, "metrics/particle/event/met_abs_error": 1.1338750123977661, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04059572145342827, "metrics/particle/event/scalar_pt_response_mean": 0.9908608198165894, "metrics/particle/event/vector_pt_closure": 0.04240339994430542, "metrics/particle/matched/delta_eta_abs_mean": 0.00756742712110281, "metrics/particle/matched/delta_phi_abs_mean": 0.006396666169166565, "metrics/particle/matched/delta_r_mean": 0.011052649468183517, "metrics/particle/matched/energy_relative_abs_mean": 0.058139681816101074, "metrics/particle/matched/pid_accuracy": 0.9850746393203735, "metrics/particle/matched/pt_relative_abs_mean": 0.057972438633441925, "metrics/particle/matching/duplicate_fraction": 0.011353711597621441, "metrics/particle/matching/efficiency": 0.9289427995681763, "metrics/particle/matching/f1": 0.9325793981552124, "metrics/particle/matching/purity": 0.936244547367096, "metrics/particle/matching/target_coverage_dr0p05": 0.9168111085891724, "metrics/particle/matching/target_coverage_dr0p10": 0.9462738037109375, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8665511012077332, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9402079582214355}, "step_train_time": 0.47477245330810547, "step_valid_time": 2.829132553999443, "step_total_time": 2250.883042573929, "elapsed_training_loop_seconds": 2250.883042573929, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9977861877935159, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02323201489961435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 428, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9953271028037384, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9918509895227008, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.004672897196261682, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9883177570093458, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9848661233993015, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.21509719900132, "step_plot_time": 3.4747522360012226, "step_validation_cycle_time": 10.838052146998962} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_2000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_2000.json new file mode 100644 index 0000000000000000000000000000000000000000..27e181b16707a5980294afcd15d5c0efab1914fb --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_2000.json @@ -0,0 +1 @@ +{"epoch": 3.5587188612099645, "train": {"Classification_binary": 1.007757306098938, "Classification": 0.0031315158121287823, "Regression_pt": 0.042948462069034576, "Regression_eta": 0.00029018474742770195, "Regression_sin_phi": 0.0007680199341848493, "Regression_cos_phi": 0.0008221978787332773, "Regression_energy": 0.042157500982284546, "Total": 3.105710029602051, "Total_unweighted": 1.0978751875227317}, "train_group": {}, "valid": {"Classification_binary": 1.0320603847503662, "Classification": 0.008527420461177826, "Regression_pt": 0.09037012606859207, "Regression_eta": 0.01195613294839859, "Regression_sin_phi": 0.0024626473896205425, "Regression_cos_phi": 0.002763918833807111, "Regression_energy": 0.09850999712944031, "Total": 8.224637985229492, "Total_unweighted": 1.2466506275814027, "metrics/particle/class_1/efficiency": 0.9698376059532166, "metrics/particle/class_1/pid_accuracy": 0.9784688949584961, "metrics/particle/class_2/efficiency": 0.43478259444236755, "metrics/particle/class_2/pid_accuracy": 0.8999999761581421, "metrics/particle/class_3/efficiency": 0.8710317611694336, "metrics/particle/class_3/pid_accuracy": 0.9908883571624756, "metrics/particle/class_4/efficiency": 0.9509803652763367, "metrics/particle/class_4/pid_accuracy": 0.9484536051750183, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.06400000303983688, "metrics/particle/count/mae": 0.23199999332427979, "metrics/particle/count/prediction_mean": 4.552000045776367, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.040318846702575684, "metrics/particle/event/energy_response_mean": 0.9872361421585083, "metrics/particle/event/met_abs_error": 1.064081072807312, "metrics/particle/event/scalar_pt_relative_abs_error": 0.038585636764764786, "metrics/particle/event/scalar_pt_response_mean": 0.9903660416603088, "metrics/particle/event/vector_pt_closure": 0.04212027043104172, "metrics/particle/matched/delta_eta_abs_mean": 0.007483605295419693, "metrics/particle/matched/delta_phi_abs_mean": 0.007167143747210503, "metrics/particle/matched/delta_r_mean": 0.011618858203291893, "metrics/particle/matched/energy_relative_abs_mean": 0.058299385011196136, "metrics/particle/matched/pid_accuracy": 0.9792060256004333, "metrics/particle/matched/pt_relative_abs_mean": 0.05930439010262489, "metrics/particle/matching/duplicate_fraction": 0.011423550546169281, "metrics/particle/matching/efficiency": 0.9168111085891724, "metrics/particle/matching/f1": 0.9232111573219299, "metrics/particle/matching/purity": 0.9297012090682983, "metrics/particle/matching/target_coverage_dr0p05": 0.909878671169281, "metrics/particle/matching/target_coverage_dr0p10": 0.9384748935699463, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8596187233924866, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9324089884757996}, "step_train_time": 0.4786083698272705, "step_valid_time": 2.8152545439997994, "step_total_time": 231.3057508468628, "elapsed_training_loop_seconds": 231.3057508468628, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9988297568182308, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02469316547158984, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 432, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9884259259259259, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9895712630359212, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.011574074074074073, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9861111111111112, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9872537659327926, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.1857174769993435, "step_plot_time": 3.40861710199988, "step_validation_cycle_time": 10.792622438999388} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..b1d6b2d9cac92bf8b69781dc9c6d413651a736c2 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 0.1690361648797989, "Classification": 2.4546467102481984e-05, "Regression_pt": 0.014835641719400883, "Regression_eta": 0.000535688828676939, "Regression_sin_phi": 0.00015054944378789514, "Regression_cos_phi": 0.00019284040899947286, "Regression_energy": 0.015784863382577896, "Total": 0.7821511626243591, "Total_unweighted": 0.20056029513034446}, "train_group": {}, "valid": {"Classification_binary": 1.381539225578308, "Classification": 0.015070699155330658, "Regression_pt": 0.08768188208341599, "Regression_eta": 0.015671007335186005, "Regression_sin_phi": 0.002696534153074026, "Regression_cos_phi": 0.002615353325381875, "Regression_energy": 0.09396307915449142, "Total": 9.726592063903809, "Total_unweighted": 1.599237780785188, "metrics/particle/class_1/efficiency": 0.9698376059532166, "metrics/particle/class_1/pid_accuracy": 0.9856459498405457, "metrics/particle/class_2/efficiency": 0.3913043439388275, "metrics/particle/class_2/pid_accuracy": 0.8888888955116272, "metrics/particle/class_3/efficiency": 0.89682537317276, "metrics/particle/class_3/pid_accuracy": 0.9933628439903259, "metrics/particle/class_4/efficiency": 0.970588207244873, "metrics/particle/class_4/pid_accuracy": 0.9696969985961914, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.957446813583374, "metrics/particle/count/bias_mean": -0.035999998450279236, "metrics/particle/count/mae": 0.2280000001192093, "metrics/particle/count/prediction_mean": 4.579999923706055, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.043056927621364594, "metrics/particle/event/energy_response_mean": 0.9992488622665405, "metrics/particle/event/met_abs_error": 1.1469824314117432, "metrics/particle/event/scalar_pt_relative_abs_error": 0.040816906839609146, "metrics/particle/event/scalar_pt_response_mean": 0.99762362241745, "metrics/particle/event/vector_pt_closure": 0.04235600680112839, "metrics/particle/matched/delta_eta_abs_mean": 0.008076942525804043, "metrics/particle/matched/delta_phi_abs_mean": 0.0065050264820456505, "metrics/particle/matched/delta_r_mean": 0.01157163642346859, "metrics/particle/matched/energy_relative_abs_mean": 0.0592804029583931, "metrics/particle/matched/pid_accuracy": 0.9841417670249939, "metrics/particle/matched/pt_relative_abs_mean": 0.05932433530688286, "metrics/particle/matching/duplicate_fraction": 0.011353711597621441, "metrics/particle/matching/efficiency": 0.9289427995681763, "metrics/particle/matching/f1": 0.9325793981552124, "metrics/particle/matching/purity": 0.936244547367096, "metrics/particle/matching/target_coverage_dr0p05": 0.9090121388435364, "metrics/particle/matching/target_coverage_dr0p10": 0.9436741471290588, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8604852557182312, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9419410824775696}, "step_train_time": 0.49659180641174316, "step_valid_time": 2.8227453819999937, "step_total_time": 2370.052114009857, "elapsed_training_loop_seconds": 2370.052114009857, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002877547894657, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.021755830636171702, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9976580796252927, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.00234192037470726, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9929742388758782, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9883449883449883, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.197105528000975, "step_plot_time": 3.4920144980005716, "step_validation_cycle_time": 10.839412891000393} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_21000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_21000.json new file mode 100644 index 0000000000000000000000000000000000000000..345794c92d8aa7218684d9f145b3ee9fca2a0601 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_21000.json @@ -0,0 +1 @@ +{"epoch": 37.36654804270463, "train": {"Classification_binary": 0.11911621689796448, "Classification": 0.00031314659281633794, "Regression_pt": 0.0947175994515419, "Regression_eta": 0.00017054197087418288, "Regression_sin_phi": 0.0002561736910138279, "Regression_cos_phi": 9.831505303736776e-05, "Regression_energy": 0.09740306437015533, "Total": 2.7784135341644287, "Total_unweighted": 0.31207505802740343}, "train_group": {}, "valid": {"Classification_binary": 1.4476207494735718, "Classification": 0.016297245398163795, "Regression_pt": 0.08665473014116287, "Regression_eta": 0.016745859757065773, "Regression_sin_phi": 0.0026761952321976423, "Regression_cos_phi": 0.0024727568961679935, "Regression_energy": 0.09172314405441284, "Total": 9.969222068786621, "Total_unweighted": 1.6641906809527427, "metrics/particle/class_1/efficiency": 0.9744779467582703, "metrics/particle/class_1/pid_accuracy": 0.9928571581840515, "metrics/particle/class_2/efficiency": 0.3913043439388275, "metrics/particle/class_2/pid_accuracy": 0.8888888955116272, "metrics/particle/class_3/efficiency": 0.8809523582458496, "metrics/particle/class_3/pid_accuracy": 0.9954954981803894, "metrics/particle/class_4/efficiency": 0.970588207244873, "metrics/particle/class_4/pid_accuracy": 0.9595959782600403, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.957446813583374, "metrics/particle/count/bias_mean": -0.04800000041723251, "metrics/particle/count/mae": 0.25600001215934753, "metrics/particle/count/prediction_mean": 4.567999839782715, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.04117434099316597, "metrics/particle/event/energy_response_mean": 0.991838812828064, "metrics/particle/event/met_abs_error": 0.9999133348464966, "metrics/particle/event/scalar_pt_relative_abs_error": 0.037385493516922, "metrics/particle/event/scalar_pt_response_mean": 0.9897754788398743, "metrics/particle/event/vector_pt_closure": 0.04064270481467247, "metrics/particle/matched/delta_eta_abs_mean": 0.006985897663980722, "metrics/particle/matched/delta_phi_abs_mean": 0.005981229245662689, "metrics/particle/matched/delta_r_mean": 0.010260729119181633, "metrics/particle/matched/energy_relative_abs_mean": 0.057865455746650696, "metrics/particle/matched/pid_accuracy": 0.9868667721748352, "metrics/particle/matched/pt_relative_abs_mean": 0.058064818382263184, "metrics/particle/matching/duplicate_fraction": 0.009632224217057228, "metrics/particle/matching/efficiency": 0.923743486404419, "metrics/particle/matching/f1": 0.9285714030265808, "metrics/particle/matching/purity": 0.9334501028060913, "metrics/particle/matching/target_coverage_dr0p05": 0.911611795425415, "metrics/particle/matching/target_coverage_dr0p10": 0.9376083016395569, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8587521910667419, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9384748935699463}, "step_train_time": 0.49108242988586426, "step_valid_time": 2.888322913000593, "step_total_time": 2489.0417201519012, "elapsed_training_loop_seconds": 2489.0417201519012, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9985678956797948, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02400778164421169, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.006993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9906759906759907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.244127255999047, "step_plot_time": 3.502358137999181, "step_validation_cycle_time": 10.967832641999848} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_22000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_22000.json new file mode 100644 index 0000000000000000000000000000000000000000..aa84244bdf0ba44c9284c9b3b6b02780c5b1e1d8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_22000.json @@ -0,0 +1 @@ +{"epoch": 39.145907473309606, "train": {"Classification_binary": 0.4047803580760956, "Classification": 6.624421803280711e-05, "Regression_pt": 0.02306264080107212, "Regression_eta": 0.00030852333293296397, "Regression_sin_phi": 0.00014823328820057213, "Regression_cos_phi": 0.00043209546129219234, "Regression_energy": 0.024454982951283455, "Total": 1.2712146043777466, "Total_unweighted": 0.4532530781289097}, "train_group": {}, "valid": {"Classification_binary": 1.5582213401794434, "Classification": 0.014294711872935295, "Regression_pt": 0.09263162314891815, "Regression_eta": 0.014502826146781445, "Regression_sin_phi": 0.003283594036474824, "Regression_cos_phi": 0.0023295236751437187, "Regression_energy": 0.09902669489383698, "Total": 10.042830467224121, "Total_unweighted": 1.7842903139535338, "metrics/particle/class_1/efficiency": 0.9721577763557434, "metrics/particle/class_1/pid_accuracy": 0.9880668520927429, "metrics/particle/class_2/efficiency": 0.3913043439388275, "metrics/particle/class_2/pid_accuracy": 0.8888888955116272, "metrics/particle/class_3/efficiency": 0.9007936716079712, "metrics/particle/class_3/pid_accuracy": 0.9933920502662659, "metrics/particle/class_4/efficiency": 0.970588207244873, "metrics/particle/class_4/pid_accuracy": 0.9696969985961914, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.03200000151991844, "metrics/particle/count/mae": 0.23999999463558197, "metrics/particle/count/prediction_mean": 4.584000110626221, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.046194322407245636, "metrics/particle/event/energy_response_mean": 0.9892491102218628, "metrics/particle/event/met_abs_error": 1.1470640897750854, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04381520673632622, "metrics/particle/event/scalar_pt_response_mean": 0.9901095032691956, "metrics/particle/event/vector_pt_closure": 0.04674852266907692, "metrics/particle/matched/delta_eta_abs_mean": 0.007506115827709436, "metrics/particle/matched/delta_phi_abs_mean": 0.007254502736032009, "metrics/particle/matched/delta_r_mean": 0.01152765192091465, "metrics/particle/matched/energy_relative_abs_mean": 0.059706512838602066, "metrics/particle/matched/pid_accuracy": 0.9860464930534363, "metrics/particle/matched/pt_relative_abs_mean": 0.059639956802129745, "metrics/particle/matching/duplicate_fraction": 0.00872600357979536, "metrics/particle/matching/efficiency": 0.9315424561500549, "metrics/particle/matching/f1": 0.9347826242446899, "metrics/particle/matching/purity": 0.9380453824996948, "metrics/particle/matching/target_coverage_dr0p05": 0.9124783277511597, "metrics/particle/matching/target_coverage_dr0p10": 0.9462738037109375, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8613518476486206, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9419410824775696}, "step_train_time": 0.49228858947753906, "step_valid_time": 2.875041422999857, "step_total_time": 2608.8680894374847, "elapsed_training_loop_seconds": 2608.8680894374847, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9982226056840449, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.024632223058486336, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9976580796252927, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.00234192037470726, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.990632318501171, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.986013986013986, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.241487356999642, "step_plot_time": 3.467545343999518, "step_validation_cycle_time": 10.905107855000097} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_23000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_23000.json new file mode 100644 index 0000000000000000000000000000000000000000..16c59acb47f5dfc5ec64e7c8d9de77851ee8cbe9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_23000.json @@ -0,0 +1 @@ +{"epoch": 40.92526690391459, "train": {"Classification_binary": 0.7498351335525513, "Classification": 0.000557185965590179, "Regression_pt": 0.041708722710609436, "Regression_eta": 0.020731540396809578, "Regression_sin_phi": 0.010216319002211094, "Regression_cos_phi": 0.0030993768014013767, "Regression_energy": 0.04241781309247017, "Total": 11.801164627075195, "Total_unweighted": 0.8685660915216431}, "train_group": {}, "valid": {"Classification_binary": 1.6854336261749268, "Classification": 0.01713448576629162, "Regression_pt": 0.08921458572149277, "Regression_eta": 0.016436824575066566, "Regression_sin_phi": 0.0030623229686170816, "Regression_cos_phi": 0.0025738459080457687, "Regression_energy": 0.09460965543985367, "Total": 10.562111854553223, "Total_unweighted": 1.9084653465542942, "metrics/particle/class_1/efficiency": 0.9698376059532166, "metrics/particle/class_1/pid_accuracy": 0.9832535982131958, "metrics/particle/class_2/efficiency": 0.3913043439388275, "metrics/particle/class_2/pid_accuracy": 0.8888888955116272, "metrics/particle/class_3/efficiency": 0.8888888955116272, "metrics/particle/class_3/pid_accuracy": 0.9910714030265808, "metrics/particle/class_4/efficiency": 0.9607843160629272, "metrics/particle/class_4/pid_accuracy": 0.9795918464660645, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.06800000369548798, "metrics/particle/count/mae": 0.2280000001192093, "metrics/particle/count/prediction_mean": 4.547999858856201, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.044006720185279846, "metrics/particle/event/energy_response_mean": 0.9921025037765503, "metrics/particle/event/met_abs_error": 1.0077351331710815, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03900330513715744, "metrics/particle/event/scalar_pt_response_mean": 0.9905627965927124, "metrics/particle/event/vector_pt_closure": 0.04121901094913483, "metrics/particle/matched/delta_eta_abs_mean": 0.007092437706887722, "metrics/particle/matched/delta_phi_abs_mean": 0.006529143080115318, "metrics/particle/matched/delta_r_mean": 0.010773075744509697, "metrics/particle/matched/energy_relative_abs_mean": 0.05740700662136078, "metrics/particle/matched/pid_accuracy": 0.9840674996376038, "metrics/particle/matched/pt_relative_abs_mean": 0.05759893357753754, "metrics/particle/matching/duplicate_fraction": 0.006156552117317915, "metrics/particle/matching/efficiency": 0.9246100783348083, "metrics/particle/matching/f1": 0.9314709901809692, "metrics/particle/matching/purity": 0.9384344816207886, "metrics/particle/matching/target_coverage_dr0p05": 0.909878671169281, "metrics/particle/matching/target_coverage_dr0p10": 0.9402079582214355, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8570190668106079, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9376083016395569}, "step_train_time": 0.47565293312072754, "step_valid_time": 2.837288234999505, "step_total_time": 2727.726704120636, "elapsed_training_loop_seconds": 2727.726704120636, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0009073163053994, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.023140721850455792, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 428, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9976635514018691, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9941792782305006, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.002336448598130841, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9929906542056075, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.989522700814901, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.2158685340000375, "step_plot_time": 3.508747838999625, "step_validation_cycle_time": 10.88335120100055} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_24000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_24000.json new file mode 100644 index 0000000000000000000000000000000000000000..0110f0d0a96b1df26e961491d189bbae6a78ddde --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_24000.json @@ -0,0 +1 @@ +{"epoch": 42.704626334519574, "train": {"Classification_binary": 0.30780094861984253, "Classification": 1.4626310075982474e-05, "Regression_pt": 0.021212738007307053, "Regression_eta": 0.000787820725236088, "Regression_sin_phi": 0.00034583554952405393, "Regression_cos_phi": 0.0010568914003670216, "Regression_energy": 0.025035589933395386, "Total": 1.5244718790054321, "Total_unweighted": 0.3562544505457481}, "train_group": {}, "valid": {"Classification_binary": 1.8809679746627808, "Classification": 0.01550939492881298, "Regression_pt": 0.09374827146530151, "Regression_eta": 0.017782039940357208, "Regression_sin_phi": 0.002851499943062663, "Regression_cos_phi": 0.0024000187404453754, "Regression_energy": 0.09824294596910477, "Total": 10.735905647277832, "Total_unweighted": 2.1115021456498653, "metrics/particle/class_1/efficiency": 0.9698376059532166, "metrics/particle/class_1/pid_accuracy": 0.9832535982131958, "metrics/particle/class_2/efficiency": 0.3478260934352875, "metrics/particle/class_2/pid_accuracy": 0.875, "metrics/particle/class_3/efficiency": 0.8829365372657776, "metrics/particle/class_3/pid_accuracy": 0.9955056309700012, "metrics/particle/class_4/efficiency": 0.970588207244873, "metrics/particle/class_4/pid_accuracy": 0.9696969985961914, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.0560000017285347, "metrics/particle/count/mae": 0.24799999594688416, "metrics/particle/count/prediction_mean": 4.559999942779541, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.04655452445149422, "metrics/particle/event/energy_response_mean": 0.9933835864067078, "metrics/particle/event/met_abs_error": 1.1525382995605469, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04265410453081131, "metrics/particle/event/scalar_pt_response_mean": 0.9911140203475952, "metrics/particle/event/vector_pt_closure": 0.04601751267910004, "metrics/particle/matched/delta_eta_abs_mean": 0.006801838055253029, "metrics/particle/matched/delta_phi_abs_mean": 0.006564146373420954, "metrics/particle/matched/delta_r_mean": 0.010513982735574245, "metrics/particle/matched/energy_relative_abs_mean": 0.05814701318740845, "metrics/particle/matched/pid_accuracy": 0.9849624037742615, "metrics/particle/matched/pt_relative_abs_mean": 0.05792764946818352, "metrics/particle/matching/duplicate_fraction": 0.009649123065173626, "metrics/particle/matching/efficiency": 0.9220104217529297, "metrics/particle/matching/f1": 0.9276373386383057, "metrics/particle/matching/purity": 0.9333333373069763, "metrics/particle/matching/target_coverage_dr0p05": 0.9055459499359131, "metrics/particle/matching/target_coverage_dr0p10": 0.9367417693138123, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8578855991363525, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9358752369880676}, "step_train_time": 0.5218973159790039, "step_valid_time": 2.843766689000404, "step_total_time": 2846.3649849891663, "elapsed_training_loop_seconds": 2846.3649849891663, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9996286441615794, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.023891780690032394, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9953379953379954, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9930232558139535, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.004662004662004662, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.986013986013986, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9837209302325581, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.220529801001248, "step_plot_time": 3.4651009009994596, "step_validation_cycle_time": 10.856208608000088} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_25000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_25000.json new file mode 100644 index 0000000000000000000000000000000000000000..d8a542ab357b484199ab4f53e82b8a6aac122c82 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_25000.json @@ -0,0 +1 @@ +{"epoch": 44.48398576512456, "train": {"Classification_binary": 0.6706582307815552, "Classification": 2.9006271233811276e-06, "Regression_pt": 0.012599888257682323, "Regression_eta": 0.003710153279826045, "Regression_sin_phi": 0.00039182580076158047, "Regression_cos_phi": 0.00017464894335716963, "Regression_energy": 0.02011687308549881, "Total": 1.911682367324829, "Total_unweighted": 0.7076545207758045}, "train_group": {}, "valid": {"Classification_binary": 1.6774958372116089, "Classification": 0.017473815008997917, "Regression_pt": 0.09381311386823654, "Regression_eta": 0.017158981412649155, "Regression_sin_phi": 0.0026431831065565348, "Regression_cos_phi": 0.002360723912715912, "Regression_energy": 0.09905384480953217, "Total": 10.498993873596191, "Total_unweighted": 1.9099994993302971, "metrics/particle/class_1/efficiency": 0.9675173759460449, "metrics/particle/class_1/pid_accuracy": 0.9928057789802551, "metrics/particle/class_2/efficiency": 0.43478259444236755, "metrics/particle/class_2/pid_accuracy": 0.8999999761581421, "metrics/particle/class_3/efficiency": 0.8948412537574768, "metrics/particle/class_3/pid_accuracy": 0.9933481216430664, "metrics/particle/class_4/efficiency": 0.9803921580314636, "metrics/particle/class_4/pid_accuracy": 0.9700000286102295, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.019999999552965164, "metrics/particle/count/mae": 0.21199999749660492, "metrics/particle/count/prediction_mean": 4.5960001945495605, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.04427628591656685, "metrics/particle/event/energy_response_mean": 0.9945485591888428, "metrics/particle/event/met_abs_error": 1.0513535737991333, "metrics/particle/event/scalar_pt_relative_abs_error": 0.039431381970644, "metrics/particle/event/scalar_pt_response_mean": 0.9926126599311829, "metrics/particle/event/vector_pt_closure": 0.042041510343551636, "metrics/particle/matched/delta_eta_abs_mean": 0.0071167657151818275, "metrics/particle/matched/delta_phi_abs_mean": 0.00671997619792819, "metrics/particle/matched/delta_r_mean": 0.010932287201285362, "metrics/particle/matched/energy_relative_abs_mean": 0.05911383405327797, "metrics/particle/matched/pid_accuracy": 0.9878731369972229, "metrics/particle/matched/pt_relative_abs_mean": 0.05949223041534424, "metrics/particle/matching/duplicate_fraction": 0.007832897827029228, "metrics/particle/matching/efficiency": 0.9289427995681763, "metrics/particle/matching/f1": 0.9309596419334412, "metrics/particle/matching/purity": 0.9329851865768433, "metrics/particle/matching/target_coverage_dr0p05": 0.9150779843330383, "metrics/particle/matching/target_coverage_dr0p10": 0.9471403956413269, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8622183799743652, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9402079582214355}, "step_train_time": 0.4777376651763916, "step_valid_time": 2.8859721699991496, "step_total_time": 2964.966162919998, "elapsed_training_loop_seconds": 2964.966162919998, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9995651962292788, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.020843039909523187, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 428, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9976635514018691, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9941792782305006, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.002336448598130841, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9906542056074766, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9871944121071012, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.2583227479990455, "step_plot_time": 3.5199729920004756, "step_validation_cycle_time": 11.000345304000803} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_26000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_26000.json new file mode 100644 index 0000000000000000000000000000000000000000..fceac491bff72c360373a954868bb653cdac237e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_26000.json @@ -0,0 +1 @@ +{"epoch": 46.263345195729535, "train": {"Classification_binary": 0.5132914185523987, "Classification": 7.160907443903852e-06, "Regression_pt": 0.017293298617005348, "Regression_eta": 0.0008032943587750196, "Regression_sin_phi": 5.968074401607737e-05, "Regression_cos_phi": 4.052207441418432e-05, "Regression_energy": 0.018163545057177544, "Total": 1.1273390054702759, "Total_unweighted": 0.5496589203112308}, "train_group": {}, "valid": {"Classification_binary": 1.8663640022277832, "Classification": 0.018901299685239792, "Regression_pt": 0.09881116449832916, "Regression_eta": 0.016969330608844757, "Regression_sin_phi": 0.002760309027507901, "Regression_cos_phi": 0.0024320262018591166, "Regression_energy": 0.10385122150182724, "Total": 11.004379272460938, "Total_unweighted": 2.110089353751391, "metrics/particle/class_1/efficiency": 0.9651972055435181, "metrics/particle/class_1/pid_accuracy": 0.9807692170143127, "metrics/particle/class_2/efficiency": 0.3913043439388275, "metrics/particle/class_2/pid_accuracy": 0.8888888955116272, "metrics/particle/class_3/efficiency": 0.9007936716079712, "metrics/particle/class_3/pid_accuracy": 0.9889867901802063, "metrics/particle/class_4/efficiency": 0.970588207244873, "metrics/particle/class_4/pid_accuracy": 0.9696969985961914, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.957446813583374, "metrics/particle/count/bias_mean": 0.004000000189989805, "metrics/particle/count/mae": 0.23600000143051147, "metrics/particle/count/prediction_mean": 4.619999885559082, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.04467249661684036, "metrics/particle/event/energy_response_mean": 0.9908575415611267, "metrics/particle/event/met_abs_error": 1.1767258644104004, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04086843132972717, "metrics/particle/event/scalar_pt_response_mean": 0.9896268248558044, "metrics/particle/event/vector_pt_closure": 0.04544174671173096, "metrics/particle/matched/delta_eta_abs_mean": 0.007357334718108177, "metrics/particle/matched/delta_phi_abs_mean": 0.006301740184426308, "metrics/particle/matched/delta_r_mean": 0.010812479071319103, "metrics/particle/matched/energy_relative_abs_mean": 0.05943846330046654, "metrics/particle/matched/pid_accuracy": 0.9804104566574097, "metrics/particle/matched/pt_relative_abs_mean": 0.05956519767642021, "metrics/particle/matching/duplicate_fraction": 0.012121211737394333, "metrics/particle/matching/efficiency": 0.9289427995681763, "metrics/particle/matching/f1": 0.9285404682159424, "metrics/particle/matching/purity": 0.9281385540962219, "metrics/particle/matching/target_coverage_dr0p05": 0.9090121388435364, "metrics/particle/matching/target_coverage_dr0p10": 0.9436741471290588, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8613518476486206, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9436741471290588}, "step_train_time": 0.4814448356628418, "step_valid_time": 2.8632372749998467, "step_total_time": 3084.7111551761627, "elapsed_training_loop_seconds": 3084.7111551761627, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9992681462045354, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.023980582228075198, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 428, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9930394431554525, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9953488372093023, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9941927990708479, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.004651162790697674, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9872241579558653, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9930394431554525, "step_test_time": 4.266888127000129, "step_plot_time": 3.4813736560008692, "step_validation_cycle_time": 10.938704205998874} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_27000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_27000.json new file mode 100644 index 0000000000000000000000000000000000000000..ddca07ee29369feba141c8f53acdcf5e4efe0568 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_27000.json @@ -0,0 +1 @@ +{"epoch": 48.04270462633452, "train": {"Classification_binary": 0.038728535175323486, "Classification": 0.00038151591434143484, "Regression_pt": 0.013499844819307327, "Regression_eta": 0.0002297700266353786, "Regression_sin_phi": 0.00023096083896234632, "Regression_cos_phi": 0.000201299277250655, "Regression_energy": 0.013039905577898026, "Total": 0.6375031471252441, "Total_unweighted": 0.06631183162971865}, "train_group": {}, "valid": {"Classification_binary": 2.0658907890319824, "Classification": 0.018529178574681282, "Regression_pt": 0.09237979352474213, "Regression_eta": 0.018168453127145767, "Regression_sin_phi": 0.002731945365667343, "Regression_cos_phi": 0.0027385696303099394, "Regression_energy": 0.09623053669929504, "Total": 11.244078636169434, "Total_unweighted": 2.296669265953824, "metrics/particle/class_1/efficiency": 0.9791183471679688, "metrics/particle/class_1/pid_accuracy": 0.9834123253822327, "metrics/particle/class_2/efficiency": 0.3913043439388275, "metrics/particle/class_2/pid_accuracy": 0.8888888955116272, "metrics/particle/class_3/efficiency": 0.8908730149269104, "metrics/particle/class_3/pid_accuracy": 0.993318498134613, "metrics/particle/class_4/efficiency": 0.9803921580314636, "metrics/particle/class_4/pid_accuracy": 0.9599999785423279, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.978723406791687, "metrics/particle/count/bias_mean": -0.012000000104308128, "metrics/particle/count/mae": 0.21199999749660492, "metrics/particle/count/prediction_mean": 4.604000091552734, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.04130210354924202, "metrics/particle/event/energy_response_mean": 0.9978424310684204, "metrics/particle/event/met_abs_error": 1.0313466787338257, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03735370561480522, "metrics/particle/event/scalar_pt_response_mean": 0.9965517520904541, "metrics/particle/event/vector_pt_closure": 0.039564941078424454, "metrics/particle/matched/delta_eta_abs_mean": 0.0073513188399374485, "metrics/particle/matched/delta_phi_abs_mean": 0.0065503292717039585, "metrics/particle/matched/delta_r_mean": 0.010950401425361633, "metrics/particle/matched/energy_relative_abs_mean": 0.06144068017601967, "metrics/particle/matched/pid_accuracy": 0.9841713309288025, "metrics/particle/matched/pt_relative_abs_mean": 0.06170213222503662, "metrics/particle/matching/duplicate_fraction": 0.010425716638565063, "metrics/particle/matching/efficiency": 0.9306759238243103, "metrics/particle/matching/f1": 0.9318872094154358, "metrics/particle/matching/purity": 0.9331016540527344, "metrics/particle/matching/target_coverage_dr0p05": 0.915944516658783, "metrics/particle/matching/target_coverage_dr0p10": 0.9462738037109375, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8596187233924866, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9445407390594482}, "step_train_time": 0.4897887706756592, "step_valid_time": 2.862391722001121, "step_total_time": 3203.489185333252, "elapsed_training_loop_seconds": 3203.489185333252, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9987789605463917, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02303188122512556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9930232558139535, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.991869918699187, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0069767441860465115, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9895470383275261, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.222498883998924, "step_plot_time": 3.4937843549996614, "step_validation_cycle_time": 10.919779726000343} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_28000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_28000.json new file mode 100644 index 0000000000000000000000000000000000000000..cd9b4c1c6344390d780d527363d7ebdfb8d4fb96 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_28000.json @@ -0,0 +1 @@ +{"epoch": 49.8220640569395, "train": {"Classification_binary": 0.013630165718495846, "Classification": 4.328619525040267e-07, "Regression_pt": 0.013025103136897087, "Regression_eta": 0.0001850652479333803, "Regression_sin_phi": 9.867490007309243e-05, "Regression_cos_phi": 9.108382073463872e-05, "Regression_energy": 0.012689382769167423, "Total": 0.4504941403865814, "Total_unweighted": 0.03971990845525397}, "train_group": {}, "valid": {"Classification_binary": 2.2161130905151367, "Classification": 0.018245359882712364, "Regression_pt": 0.09384358674287796, "Regression_eta": 0.01853281818330288, "Regression_sin_phi": 0.00273092626594007, "Regression_cos_phi": 0.0024665261153131723, "Regression_energy": 0.09731072932481766, "Total": 11.3729248046875, "Total_unweighted": 2.449243037030101, "metrics/particle/class_1/efficiency": 0.9721577763557434, "metrics/particle/class_1/pid_accuracy": 0.9856801629066467, "metrics/particle/class_2/efficiency": 0.3478260934352875, "metrics/particle/class_2/pid_accuracy": 0.875, "metrics/particle/class_3/efficiency": 0.8888888955116272, "metrics/particle/class_3/pid_accuracy": 0.9933035969734192, "metrics/particle/class_4/efficiency": 0.9607843160629272, "metrics/particle/class_4/pid_accuracy": 0.9693877696990967, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9893617033958435, "metrics/particle/count/bias_mean": -0.019999999552965164, "metrics/particle/count/mae": 0.24400000274181366, "metrics/particle/count/prediction_mean": 4.5960001945495605, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.04757404699921608, "metrics/particle/event/energy_response_mean": 0.994758129119873, "metrics/particle/event/met_abs_error": 1.1601167917251587, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04079326614737511, "metrics/particle/event/scalar_pt_response_mean": 0.9908852577209473, "metrics/particle/event/vector_pt_closure": 0.04511529207229614, "metrics/particle/matched/delta_eta_abs_mean": 0.006791488733142614, "metrics/particle/matched/delta_phi_abs_mean": 0.007376543711870909, "metrics/particle/matched/delta_r_mean": 0.011156518943607807, "metrics/particle/matched/energy_relative_abs_mean": 0.05863377824425697, "metrics/particle/matched/pid_accuracy": 0.9868791103363037, "metrics/particle/matched/pt_relative_abs_mean": 0.05841211974620819, "metrics/particle/matching/duplicate_fraction": 0.0121845081448555, "metrics/particle/matching/efficiency": 0.9246100783348083, "metrics/particle/matching/f1": 0.926617443561554, "metrics/particle/matching/purity": 0.9286335706710815, "metrics/particle/matching/target_coverage_dr0p05": 0.9107452630996704, "metrics/particle/matching/target_coverage_dr0p10": 0.941074550151825, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8630849123001099, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9402079582214355}, "step_train_time": 0.48391127586364746, "step_valid_time": 2.871595039001477, "step_total_time": 3322.3150618076324, "elapsed_training_loop_seconds": 3322.3150618076324, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9977028869811961, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.022787871472778054, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 428, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 422, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9976635514018691, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9941792782305006, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.002336448598130841, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9791183294663574, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.985981308411215, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9825378346915018, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.238114173000213, "step_plot_time": 3.5154128640006093, "step_validation_cycle_time": 10.960672283999884} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_29000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_29000.json new file mode 100644 index 0000000000000000000000000000000000000000..fac134c624ee1fbbd504b9d7e8e3640bc20690ac --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_29000.json @@ -0,0 +1 @@ +{"epoch": 51.60142348754449, "train": {"Classification_binary": 0.018781736493110657, "Classification": 3.9905207813717425e-06, "Regression_pt": 0.007353045046329498, "Regression_eta": 0.0001572042820043862, "Regression_sin_phi": 5.4684853239450604e-05, "Regression_cos_phi": 7.26835714885965e-05, "Regression_energy": 0.007945687510073185, "Total": 0.28901195526123047, "Total_unweighted": 0.034369032277027145}, "train_group": {}, "valid": {"Classification_binary": 2.4496543407440186, "Classification": 0.02043871022760868, "Regression_pt": 0.09468044340610504, "Regression_eta": 0.019955646246671677, "Regression_sin_phi": 0.0030476837418973446, "Regression_cos_phi": 0.0021960067097097635, "Regression_energy": 0.09776655584573746, "Total": 12.139330863952637, "Total_unweighted": 2.6877393869217485, "metrics/particle/class_1/efficiency": 0.9698376059532166, "metrics/particle/class_1/pid_accuracy": 0.9832535982131958, "metrics/particle/class_2/efficiency": 0.30434781312942505, "metrics/particle/class_2/pid_accuracy": 0.8571428656578064, "metrics/particle/class_3/efficiency": 0.8849206566810608, "metrics/particle/class_3/pid_accuracy": 0.9955157041549683, "metrics/particle/class_4/efficiency": 0.9607843160629272, "metrics/particle/class_4/pid_accuracy": 0.9693877696990967, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9468085169792175, "metrics/particle/count/bias_mean": -0.019999999552965164, "metrics/particle/count/mae": 0.25999999046325684, "metrics/particle/count/prediction_mean": 4.5960001945495605, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.046966515481472015, "metrics/particle/event/energy_response_mean": 0.9887477159500122, "metrics/particle/event/met_abs_error": 1.2100856304168701, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04555699974298477, "metrics/particle/event/scalar_pt_response_mean": 0.9887147545814514, "metrics/particle/event/vector_pt_closure": 0.04661388322710991, "metrics/particle/matched/delta_eta_abs_mean": 0.0069414181634783745, "metrics/particle/matched/delta_phi_abs_mean": 0.005928549915552139, "metrics/particle/matched/delta_r_mean": 0.010271194390952587, "metrics/particle/matched/energy_relative_abs_mean": 0.05890439823269844, "metrics/particle/matched/pid_accuracy": 0.9830667972564697, "metrics/particle/matched/pt_relative_abs_mean": 0.059186603873968124, "metrics/particle/matching/duplicate_fraction": 0.013925151899456978, "metrics/particle/matching/efficiency": 0.9211438298225403, "metrics/particle/matching/f1": 0.923143744468689, "metrics/particle/matching/purity": 0.9251523017883301, "metrics/particle/matching/target_coverage_dr0p05": 0.9090121388435364, "metrics/particle/matching/target_coverage_dr0p10": 0.9402079582214355, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8561525344848633, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9367417693138123}, "step_train_time": 0.480513334274292, "step_valid_time": 2.8250993490000837, "step_total_time": 3441.136437892914, "elapsed_training_loop_seconds": 3441.136437892914, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9962844349746762, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.023974139286297325, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9953596287703016, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9976744186046511, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9965156794425087, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.002325581395348837, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9895470383275261, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9953596287703016, "step_test_time": 4.2550352120015305, "step_plot_time": 3.4869503659992915, "step_validation_cycle_time": 10.910321246999956} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_3000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_3000.json new file mode 100644 index 0000000000000000000000000000000000000000..ea12c735428a5f614081059dc51ad9307e42e5bc --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_3000.json @@ -0,0 +1 @@ +{"epoch": 5.338078291814947, "train": {"Classification_binary": 0.5119394063949585, "Classification": 0.005427062977105379, "Regression_pt": 0.053701624274253845, "Regression_eta": 0.0014337170869112015, "Regression_sin_phi": 0.0002128190826624632, "Regression_cos_phi": 0.0004367670335341245, "Regression_energy": 0.04644985496997833, "Total": 2.743527889251709, "Total_unweighted": 0.6196012518194038}, "train_group": {}, "valid": {"Classification_binary": 1.0586377382278442, "Classification": 0.008461895398795605, "Regression_pt": 0.09220100194215775, "Regression_eta": 0.011804956011474133, "Regression_sin_phi": 0.002468493301421404, "Regression_cos_phi": 0.002563073765486479, "Regression_energy": 0.09982126206159592, "Total": 8.206412315368652, "Total_unweighted": 1.2759584207087755, "metrics/particle/class_1/efficiency": 0.9675173759460449, "metrics/particle/class_1/pid_accuracy": 0.9856114983558655, "metrics/particle/class_2/efficiency": 0.3913043439388275, "metrics/particle/class_2/pid_accuracy": 0.8888888955116272, "metrics/particle/class_3/efficiency": 0.886904776096344, "metrics/particle/class_3/pid_accuracy": 0.9888142943382263, "metrics/particle/class_4/efficiency": 0.9509803652763367, "metrics/particle/class_4/pid_accuracy": 0.9587628841400146, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.978723406791687, "metrics/particle/count/bias_mean": -0.03999999910593033, "metrics/particle/count/mae": 0.2160000056028366, "metrics/particle/count/prediction_mean": 4.576000213623047, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.03828214854001999, "metrics/particle/event/energy_response_mean": 0.985758364200592, "metrics/particle/event/met_abs_error": 1.0176901817321777, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03758621960878372, "metrics/particle/event/scalar_pt_response_mean": 0.9873279929161072, "metrics/particle/event/vector_pt_closure": 0.03830713778734207, "metrics/particle/matched/delta_eta_abs_mean": 0.007411166559904814, "metrics/particle/matched/delta_phi_abs_mean": 0.0068045323714613914, "metrics/particle/matched/delta_r_mean": 0.01120629534125328, "metrics/particle/matched/energy_relative_abs_mean": 0.05655407905578613, "metrics/particle/matched/pid_accuracy": 0.9830827116966248, "metrics/particle/matched/pt_relative_abs_mean": 0.05757245793938637, "metrics/particle/matching/duplicate_fraction": 0.014860140159726143, "metrics/particle/matching/efficiency": 0.9220104217529297, "metrics/particle/matching/f1": 0.9260226488113403, "metrics/particle/matching/purity": 0.9300699234008789, "metrics/particle/matching/target_coverage_dr0p05": 0.9185441732406616, "metrics/particle/matching/target_coverage_dr0p10": 0.9454072713851929, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8674176931381226, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.933275580406189}, "step_train_time": 0.4629356861114502, "step_valid_time": 2.7949589410000044, "step_total_time": 349.36368560791016, "elapsed_training_loop_seconds": 349.36368560791016, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9980076938581584, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02338093061463753, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9930232558139535, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.991869918699187, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0069767441860465115, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9930232558139535, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.991869918699187, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.1651872170004935, "step_plot_time": 3.4365943060001882, "step_validation_cycle_time": 10.773233864999384} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..9349b862688013148daa83864446a02dba67d5ed --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 0.07290390878915787, "Classification": 4.113558826901453e-09, "Regression_pt": 0.008770270273089409, "Regression_eta": 0.0006373712094500661, "Regression_sin_phi": 7.886239473009482e-05, "Regression_cos_phi": 0.0001620512775843963, "Regression_energy": 0.0082301776856184, "Total": 0.4760633707046509, "Total_unweighted": 0.09078264574318906}, "train_group": {}, "valid": {"Classification_binary": 2.216700553894043, "Classification": 0.01934557966887951, "Regression_pt": 0.09541455656290054, "Regression_eta": 0.019137965515255928, "Regression_sin_phi": 0.0032566525042057037, "Regression_cos_phi": 0.0022781051229685545, "Regression_energy": 0.09838911890983582, "Total": 11.848159790039062, "Total_unweighted": 2.454522532178089, "metrics/particle/class_1/efficiency": 0.9814385175704956, "metrics/particle/class_1/pid_accuracy": 0.978723406791687, "metrics/particle/class_2/efficiency": 0.3478260934352875, "metrics/particle/class_2/pid_accuracy": 0.75, "metrics/particle/class_3/efficiency": 0.9067460298538208, "metrics/particle/class_3/pid_accuracy": 0.9890590906143188, "metrics/particle/class_4/efficiency": 0.970588207244873, "metrics/particle/class_4/pid_accuracy": 0.9696969985961914, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": 0.052000001072883606, "metrics/particle/count/mae": 0.24400000274181366, "metrics/particle/count/prediction_mean": 4.668000221252441, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.04503796622157097, "metrics/particle/event/energy_response_mean": 0.9984121322631836, "metrics/particle/event/met_abs_error": 1.1693730354309082, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03938111662864685, "metrics/particle/event/scalar_pt_response_mean": 0.9930427074432373, "metrics/particle/event/vector_pt_closure": 0.04468045383691788, "metrics/particle/matched/delta_eta_abs_mean": 0.008590045385062695, "metrics/particle/matched/delta_phi_abs_mean": 0.006711704656481743, "metrics/particle/matched/delta_r_mean": 0.012158102355897427, "metrics/particle/matched/energy_relative_abs_mean": 0.061932940036058426, "metrics/particle/matched/pid_accuracy": 0.9796484708786011, "metrics/particle/matched/pt_relative_abs_mean": 0.0614420622587204, "metrics/particle/matching/duplicate_fraction": 0.014567266218364239, "metrics/particle/matching/efficiency": 0.9367417693138123, "metrics/particle/matching/f1": 0.9314950704574585, "metrics/particle/matching/purity": 0.9263067841529846, "metrics/particle/matching/target_coverage_dr0p05": 0.9107452630996704, "metrics/particle/matching/target_coverage_dr0p10": 0.9558058977127075, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8587521910667419, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9480069279670715}, "step_train_time": 0.49614381790161133, "step_valid_time": 2.859735558999091, "step_total_time": 3560.9023518562317, "elapsed_training_loop_seconds": 3560.9023518562317, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9986799720172665, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.021019721101473432, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 428, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9930394431554525, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9976689976689976, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9953488372093023, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.002331002331002331, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9906759906759907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9930394431554525, "step_test_time": 4.258096805000605, "step_plot_time": 3.546393986000112, "step_validation_cycle_time": 10.999903961999735} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_31000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_31000.json new file mode 100644 index 0000000000000000000000000000000000000000..d08fe712d5ee5b730110a94a534ebf252406507d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_31000.json @@ -0,0 +1 @@ +{"epoch": 55.16014234875445, "train": {"Classification_binary": 0.07103966176509857, "Classification": 4.3938762246398255e-06, "Regression_pt": 0.010926676914095879, "Regression_eta": 0.0002831330639310181, "Regression_sin_phi": 0.0003526044893078506, "Regression_cos_phi": 0.00029761134646832943, "Regression_energy": 0.010682087391614914, "Total": 0.6801213622093201, "Total_unweighted": 0.0935861688467412}, "train_group": {}, "valid": {"Classification_binary": 2.357863187789917, "Classification": 0.021753432229161263, "Regression_pt": 0.10601287335157394, "Regression_eta": 0.017576036974787712, "Regression_sin_phi": 0.00269296090118587, "Regression_cos_phi": 0.002471735468134284, "Regression_energy": 0.10974378883838654, "Total": 11.986254692077637, "Total_unweighted": 2.6181140155531466, "metrics/particle/class_1/efficiency": 0.9605568647384644, "metrics/particle/class_1/pid_accuracy": 0.987922728061676, "metrics/particle/class_2/efficiency": 0.3478260934352875, "metrics/particle/class_2/pid_accuracy": 0.875, "metrics/particle/class_3/efficiency": 0.8849206566810608, "metrics/particle/class_3/pid_accuracy": 0.9910314083099365, "metrics/particle/class_4/efficiency": 0.9607843160629272, "metrics/particle/class_4/pid_accuracy": 0.9693877696990967, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.978723406791687, "metrics/particle/count/bias_mean": -0.035999998450279236, "metrics/particle/count/mae": 0.23600000143051147, "metrics/particle/count/prediction_mean": 4.579999923706055, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.04872296750545502, "metrics/particle/event/energy_response_mean": 0.990064263343811, "metrics/particle/event/met_abs_error": 1.2192282676696777, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04259299114346504, "metrics/particle/event/scalar_pt_response_mean": 0.9864313006401062, "metrics/particle/event/vector_pt_closure": 0.04739229381084442, "metrics/particle/matched/delta_eta_abs_mean": 0.006966841872781515, "metrics/particle/matched/delta_phi_abs_mean": 0.006648901384323835, "metrics/particle/matched/delta_r_mean": 0.010753519833087921, "metrics/particle/matched/energy_relative_abs_mean": 0.05848323553800583, "metrics/particle/matched/pid_accuracy": 0.9858490824699402, "metrics/particle/matched/pt_relative_abs_mean": 0.05869732424616814, "metrics/particle/matching/duplicate_fraction": 0.0122270742431283, "metrics/particle/matching/efficiency": 0.9185441732406616, "metrics/particle/matching/f1": 0.9221400618553162, "metrics/particle/matching/purity": 0.9257642030715942, "metrics/particle/matching/target_coverage_dr0p05": 0.9124783277511597, "metrics/particle/matching/target_coverage_dr0p10": 0.9367417693138123, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8526862859725952, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.933275580406189}, "step_train_time": 0.520728588104248, "step_valid_time": 2.837760205999075, "step_total_time": 3680.6770219802856, "elapsed_training_loop_seconds": 3680.6770219802856, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9944534146482951, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.01906468554098195, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9953379953379954, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9930232558139535, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.004662004662004662, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.986013986013986, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9837209302325581, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.218849676999525, "step_plot_time": 3.495700293000482, "step_validation_cycle_time": 10.933641207000619} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_32000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_32000.json new file mode 100644 index 0000000000000000000000000000000000000000..dbf0f0868b93a35685cf86d61b0082c9f1c6bdf4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_32000.json @@ -0,0 +1 @@ +{"epoch": 56.93950177935943, "train": {"Classification_binary": 0.14118286967277527, "Classification": 4.3197378545301035e-05, "Regression_pt": 0.021687038242816925, "Regression_eta": 0.006999276578426361, "Regression_sin_phi": 0.0002037177764577791, "Regression_cos_phi": 0.00023289312957786024, "Regression_energy": 0.02547611854970455, "Total": 1.9630622863769531, "Total_unweighted": 0.19582511132830405}, "train_group": {}, "valid": {"Classification_binary": 2.403679847717285, "Classification": 0.02504860609769821, "Regression_pt": 0.10450025647878647, "Regression_eta": 0.02063709869980812, "Regression_sin_phi": 0.002705732826143503, "Regression_cos_phi": 0.0025603394024074078, "Regression_energy": 0.10569693893194199, "Total": 12.751827239990234, "Total_unweighted": 2.664828820154071, "metrics/particle/class_1/efficiency": 0.9651972055435181, "metrics/particle/class_1/pid_accuracy": 0.9831730723381042, "metrics/particle/class_2/efficiency": 0.3478260934352875, "metrics/particle/class_2/pid_accuracy": 0.875, "metrics/particle/class_3/efficiency": 0.9067460298538208, "metrics/particle/class_3/pid_accuracy": 0.9890590906143188, "metrics/particle/class_4/efficiency": 0.9509803652763367, "metrics/particle/class_4/pid_accuracy": 0.9896907210350037, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.936170220375061, "metrics/particle/count/bias_mean": 0.004000000189989805, "metrics/particle/count/mae": 0.21199999749660492, "metrics/particle/count/prediction_mean": 4.619999885559082, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.04348723962903023, "metrics/particle/event/energy_response_mean": 0.9904724359512329, "metrics/particle/event/met_abs_error": 1.1320593357086182, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03962435573339462, "metrics/particle/event/scalar_pt_response_mean": 0.9879727959632874, "metrics/particle/event/vector_pt_closure": 0.042568545788526535, "metrics/particle/matched/delta_eta_abs_mean": 0.007629101164638996, "metrics/particle/matched/delta_phi_abs_mean": 0.006214525084942579, "metrics/particle/matched/delta_r_mean": 0.010923073627054691, "metrics/particle/matched/energy_relative_abs_mean": 0.06056783348321915, "metrics/particle/matched/pid_accuracy": 0.9813432693481445, "metrics/particle/matched/pt_relative_abs_mean": 0.060808517038822174, "metrics/particle/matching/duplicate_fraction": 0.012987012974917889, "metrics/particle/matching/efficiency": 0.9289427995681763, "metrics/particle/matching/f1": 0.9285404682159424, "metrics/particle/matching/purity": 0.9281385540962219, "metrics/particle/matching/target_coverage_dr0p05": 0.917677640914917, "metrics/particle/matching/target_coverage_dr0p10": 0.9506065845489502, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8544194102287292, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.941074550151825}, "step_train_time": 0.49553561210632324, "step_valid_time": 2.8960371110006236, "step_total_time": 3800.4367020130157, "elapsed_training_loop_seconds": 3800.4367020130157, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9971492053244893, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02166555325975661, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 424, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 422, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9837587006960556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9929742388758782, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9883449883449883, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.00702576112412178, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9791183294663574, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9882903981264637, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9836829836829837, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9837587006960556, "step_test_time": 4.288537140000699, "step_plot_time": 3.5943046569991566, "step_validation_cycle_time": 11.114886634999493} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_33000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_33000.json new file mode 100644 index 0000000000000000000000000000000000000000..af6d1d32735d3f58ef71c375266cc3e9a6dac599 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_33000.json @@ -0,0 +1 @@ +{"epoch": 58.718861209964416, "train": {"Classification_binary": 0.18831394612789154, "Classification": 1.2119966186219244e-06, "Regression_pt": 0.035178590565919876, "Regression_eta": 0.00030433974461629987, "Regression_sin_phi": 0.00012214585149195045, "Regression_cos_phi": 8.234701817855239e-05, "Regression_energy": 0.03552747145295143, "Total": 1.2222821712493896, "Total_unweighted": 0.2595300527576683}, "train_group": {}, "valid": {"Classification_binary": 2.404873847961426, "Classification": 0.018503127619624138, "Regression_pt": 0.09842351078987122, "Regression_eta": 0.02044207789003849, "Regression_sin_phi": 0.0028688565362244844, "Regression_cos_phi": 0.0025818583089858294, "Regression_energy": 0.10143086314201355, "Total": 12.092283248901367, "Total_unweighted": 2.6491241422481835, "metrics/particle/class_1/efficiency": 0.9651972055435181, "metrics/particle/class_1/pid_accuracy": 0.9879807829856873, "metrics/particle/class_2/efficiency": 0.3478260934352875, "metrics/particle/class_2/pid_accuracy": 0.875, "metrics/particle/class_3/efficiency": 0.9007936716079712, "metrics/particle/class_3/pid_accuracy": 0.9933920502662659, "metrics/particle/class_4/efficiency": 0.9509803652763367, "metrics/particle/class_4/pid_accuracy": 0.9793814420700073, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": 0.03200000151991844, "metrics/particle/count/mae": 0.2240000069141388, "metrics/particle/count/prediction_mean": 4.6479997634887695, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.04971091449260712, "metrics/particle/event/energy_response_mean": 1.0002614259719849, "metrics/particle/event/met_abs_error": 1.2797675132751465, "metrics/particle/event/scalar_pt_relative_abs_error": 0.044837336987257004, "metrics/particle/event/scalar_pt_response_mean": 0.9964196681976318, "metrics/particle/event/vector_pt_closure": 0.04959113150835037, "metrics/particle/matched/delta_eta_abs_mean": 0.007892685011029243, "metrics/particle/matched/delta_phi_abs_mean": 0.0068694548681378365, "metrics/particle/matched/delta_r_mean": 0.011706637218594551, "metrics/particle/matched/energy_relative_abs_mean": 0.06224651634693146, "metrics/particle/matched/pid_accuracy": 0.9869036674499512, "metrics/particle/matched/pt_relative_abs_mean": 0.0624198243021965, "metrics/particle/matching/duplicate_fraction": 0.015490533784031868, "metrics/particle/matching/efficiency": 0.9263431429862976, "metrics/particle/matching/f1": 0.9231433272361755, "metrics/particle/matching/purity": 0.9199655652046204, "metrics/particle/matching/target_coverage_dr0p05": 0.919410765171051, "metrics/particle/matching/target_coverage_dr0p10": 0.9462738037109375, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8457539081573486, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.941074550151825}, "step_train_time": 0.5057525634765625, "step_valid_time": 2.8781211490004353, "step_total_time": 3919.9968638420105, "elapsed_training_loop_seconds": 3919.9968638420105, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0012851115251278, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.022294491166203856, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 421, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9953379953379954, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9930232558139535, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.004662004662004662, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9767981438515081, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9813519813519813, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9790697674418605, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.245389280000381, "step_plot_time": 3.523705860001428, "step_validation_cycle_time": 10.976733621000676} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_34000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_34000.json new file mode 100644 index 0000000000000000000000000000000000000000..7ba356eb83487b6b2164aad23eedd66150e8e667 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_34000.json @@ -0,0 +1 @@ +{"epoch": 60.49822064056939, "train": {"Classification_binary": 0.020397495478391647, "Classification": 9.340841643279418e-06, "Regression_pt": 0.009510206989943981, "Regression_eta": 0.0003421572328079492, "Regression_sin_phi": 0.0001044740856741555, "Regression_cos_phi": 8.752527355682105e-05, "Regression_energy": 0.009073088876903057, "Total": 0.3935505747795105, "Total_unweighted": 0.03952428877892089}, "train_group": {}, "valid": {"Classification_binary": 2.6329994201660156, "Classification": 0.018800942227244377, "Regression_pt": 0.10295344144105911, "Regression_eta": 0.019009876996278763, "Regression_sin_phi": 0.002637207042425871, "Regression_cos_phi": 0.002831355668604374, "Regression_energy": 0.10580670833587646, "Total": 12.187692642211914, "Total_unweighted": 2.8850389518775046, "metrics/particle/class_1/efficiency": 0.9698376059532166, "metrics/particle/class_1/pid_accuracy": 0.9880383014678955, "metrics/particle/class_2/efficiency": 0.3478260934352875, "metrics/particle/class_2/pid_accuracy": 0.875, "metrics/particle/class_3/efficiency": 0.8888888955116272, "metrics/particle/class_3/pid_accuracy": 0.9910714030265808, "metrics/particle/class_4/efficiency": 0.9607843160629272, "metrics/particle/class_4/pid_accuracy": 0.9693877696990967, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.957446813583374, "metrics/particle/count/bias_mean": -0.012000000104308128, "metrics/particle/count/mae": 0.24400000274181366, "metrics/particle/count/prediction_mean": 4.604000091552734, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.044709984213113785, "metrics/particle/event/energy_response_mean": 0.9850071668624878, "metrics/particle/event/met_abs_error": 1.1108852624893188, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04083581268787384, "metrics/particle/event/scalar_pt_response_mean": 0.9823338389396667, "metrics/particle/event/vector_pt_closure": 0.042822688817977905, "metrics/particle/matched/delta_eta_abs_mean": 0.007189137861132622, "metrics/particle/matched/delta_phi_abs_mean": 0.006575188599526882, "metrics/particle/matched/delta_r_mean": 0.010901303961873055, "metrics/particle/matched/energy_relative_abs_mean": 0.0602254755795002, "metrics/particle/matched/pid_accuracy": 0.9840525388717651, "metrics/particle/matched/pt_relative_abs_mean": 0.06057783588767052, "metrics/particle/matching/duplicate_fraction": 0.01303214579820633, "metrics/particle/matching/efficiency": 0.923743486404419, "metrics/particle/matching/f1": 0.9249457716941833, "metrics/particle/matching/purity": 0.9261511564254761, "metrics/particle/matching/target_coverage_dr0p05": 0.9107452630996704, "metrics/particle/matching/target_coverage_dr0p10": 0.9402079582214355, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8552859425544739, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9384748935699463}, "step_train_time": 0.49211621284484863, "step_valid_time": 2.864455901000838, "step_total_time": 4038.9869265556335, "elapsed_training_loop_seconds": 4038.9869265556335, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9980873296563368, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.031709340892058346, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 428, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9930394431554525, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9953488372093023, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9941927990708479, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.004651162790697674, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9930232558139535, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.991869918699187, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9930394431554525, "step_test_time": 4.2461892450010055, "step_plot_time": 3.500405414999477, "step_validation_cycle_time": 10.940343747999577} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_35000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_35000.json new file mode 100644 index 0000000000000000000000000000000000000000..8b440805d3fedb8981ee68acea022ebb882cad57 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_35000.json @@ -0,0 +1 @@ +{"epoch": 62.27758007117438, "train": {"Classification_binary": 0.06197857856750488, "Classification": 0.00011920743418158963, "Regression_pt": 0.011006961576640606, "Regression_eta": 0.0003484994813334197, "Regression_sin_phi": 0.0004418069147504866, "Regression_cos_phi": 0.0007617702358402312, "Regression_energy": 0.010938114486634731, "Total": 0.8852450847625732, "Total_unweighted": 0.08559493869688595}, "train_group": {}, "valid": {"Classification_binary": 2.6635000705718994, "Classification": 0.024311518296599388, "Regression_pt": 0.0967506617307663, "Regression_eta": 0.020259616896510124, "Regression_sin_phi": 0.0027317299973219633, "Regression_cos_phi": 0.002568861935287714, "Regression_energy": 0.09965421259403229, "Total": 12.731136322021484, "Total_unweighted": 2.909776672022417, "metrics/particle/class_1/efficiency": 0.9512761235237122, "metrics/particle/class_1/pid_accuracy": 0.9780488014221191, "metrics/particle/class_2/efficiency": 0.3913043439388275, "metrics/particle/class_2/pid_accuracy": 0.8888888955116272, "metrics/particle/class_3/efficiency": 0.8670634627342224, "metrics/particle/class_3/pid_accuracy": 0.9954233169555664, "metrics/particle/class_4/efficiency": 0.9607843160629272, "metrics/particle/class_4/pid_accuracy": 0.9897959232330322, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.957446813583374, "metrics/particle/count/bias_mean": -0.14000000059604645, "metrics/particle/count/mae": 0.2919999957084656, "metrics/particle/count/prediction_mean": 4.47599983215332, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.047715313732624054, "metrics/particle/event/energy_response_mean": 0.9854750633239746, "metrics/particle/event/met_abs_error": 1.1850364208221436, "metrics/particle/event/scalar_pt_relative_abs_error": 0.044465769082307816, "metrics/particle/event/scalar_pt_response_mean": 0.9839433431625366, "metrics/particle/event/vector_pt_closure": 0.04587658494710922, "metrics/particle/matched/delta_eta_abs_mean": 0.006750569213181734, "metrics/particle/matched/delta_phi_abs_mean": 0.007451545912772417, "metrics/particle/matched/delta_r_mean": 0.011303404346108437, "metrics/particle/matched/energy_relative_abs_mean": 0.057816196233034134, "metrics/particle/matched/pid_accuracy": 0.9837786555290222, "metrics/particle/matched/pt_relative_abs_mean": 0.05783882364630699, "metrics/particle/matching/duplicate_fraction": 0.014298480935394764, "metrics/particle/matching/efficiency": 0.9081456065177917, "metrics/particle/matching/f1": 0.9221293330192566, "metrics/particle/matching/purity": 0.936550498008728, "metrics/particle/matching/target_coverage_dr0p05": 0.8968803882598877, "metrics/particle/matching/target_coverage_dr0p10": 0.927209734916687, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8492200970649719, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9202772974967957}, "step_train_time": 0.5167624950408936, "step_valid_time": 2.8231797280004685, "step_total_time": 4157.459872484207, "elapsed_training_loop_seconds": 4157.459872484207, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000198110902126, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02481636211258742, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 423, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9953161592505855, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9906759906759907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.00468384074941452, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9814385150812065, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.990632318501171, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.986013986013986, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9860788863109049, "step_test_time": 4.238496752999708, "step_plot_time": 3.5074322640011815, "step_validation_cycle_time": 10.945163201000469} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_36000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_36000.json new file mode 100644 index 0000000000000000000000000000000000000000..e39d87e64418974c46ffa6b057835015aaec8149 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_36000.json @@ -0,0 +1 @@ +{"epoch": 64.05693950177935, "train": {"Classification_binary": 0.1172579973936081, "Classification": 2.3692181549961333e-09, "Regression_pt": 0.007891549728810787, "Regression_eta": 6.665427645202726e-05, "Regression_sin_phi": 6.136904994491488e-05, "Regression_cos_phi": 0.0004177086229901761, "Regression_energy": 0.008044667541980743, "Total": 0.48816078901290894, "Total_unweighted": 0.1337399489830049}, "train_group": {}, "valid": {"Classification_binary": 2.579207420349121, "Classification": 0.021301962435245514, "Regression_pt": 0.0959208533167839, "Regression_eta": 0.01766788214445114, "Regression_sin_phi": 0.0028096570167690516, "Regression_cos_phi": 0.002754391171038151, "Regression_energy": 0.0998007208108902, "Total": 12.078362464904785, "Total_unweighted": 2.819462887244299, "metrics/particle/class_1/efficiency": 0.9628770351409912, "metrics/particle/class_1/pid_accuracy": 0.9855421781539917, "metrics/particle/class_2/efficiency": 0.3478260934352875, "metrics/particle/class_2/pid_accuracy": 0.75, "metrics/particle/class_3/efficiency": 0.8908730149269104, "metrics/particle/class_3/pid_accuracy": 0.9977728128433228, "metrics/particle/class_4/efficiency": 0.9607843160629272, "metrics/particle/class_4/pid_accuracy": 0.9693877696990967, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.978723406791687, "metrics/particle/count/bias_mean": -0.04800000041723251, "metrics/particle/count/mae": 0.23199999332427979, "metrics/particle/count/prediction_mean": 4.567999839782715, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.04506932944059372, "metrics/particle/event/energy_response_mean": 0.9897844791412354, "metrics/particle/event/met_abs_error": 1.098703384399414, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04040247201919556, "metrics/particle/event/scalar_pt_response_mean": 0.9879805445671082, "metrics/particle/event/vector_pt_closure": 0.043275196105241776, "metrics/particle/matched/delta_eta_abs_mean": 0.00656376127153635, "metrics/particle/matched/delta_phi_abs_mean": 0.006515171378850937, "metrics/particle/matched/delta_r_mean": 0.010386333800852299, "metrics/particle/matched/energy_relative_abs_mean": 0.05990741029381752, "metrics/particle/matched/pid_accuracy": 0.9868420958518982, "metrics/particle/matched/pt_relative_abs_mean": 0.060136012732982635, "metrics/particle/matching/duplicate_fraction": 0.011383537203073502, "metrics/particle/matching/efficiency": 0.9220104217529297, "metrics/particle/matching/f1": 0.9268292784690857, "metrics/particle/matching/purity": 0.9316987991333008, "metrics/particle/matching/target_coverage_dr0p05": 0.9150779843330383, "metrics/particle/matching/target_coverage_dr0p10": 0.9428076148033142, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8483535647392273, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.933275580406189}, "step_train_time": 0.48320555686950684, "step_valid_time": 2.8418934019991866, "step_total_time": 4276.28072476387, "elapsed_training_loop_seconds": 4276.28072476387, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9980753229444376, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.022387046950256173, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 428, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9976635514018691, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9941792782305006, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.002336448598130841, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9929906542056075, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.989522700814901, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.277711377999367, "step_plot_time": 3.504992692000087, "step_validation_cycle_time": 10.960006228000566} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_37000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_37000.json new file mode 100644 index 0000000000000000000000000000000000000000..511fa38a2204386234a28c027f78e0cc39e76bef --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_37000.json @@ -0,0 +1 @@ +{"epoch": 65.83629893238434, "train": {"Classification_binary": 0.044448766857385635, "Classification": 8.396967814405798e-07, "Regression_pt": 0.010288475081324577, "Regression_eta": 0.00013709228369407356, "Regression_sin_phi": 5.079726179246791e-05, "Regression_cos_phi": 5.7929100876208395e-05, "Regression_energy": 0.010288500227034092, "Total": 0.3715576231479645, "Total_unweighted": 0.0652724005088885}, "train_group": {}, "valid": {"Classification_binary": 2.542145252227783, "Classification": 0.01943223923444748, "Regression_pt": 0.09588136523962021, "Regression_eta": 0.01776912808418274, "Regression_sin_phi": 0.0028422647155821323, "Regression_cos_phi": 0.0021759478840976954, "Regression_energy": 0.09973960369825363, "Total": 11.730254173278809, "Total_unweighted": 2.779985801083967, "metrics/particle/class_1/efficiency": 0.9651972055435181, "metrics/particle/class_1/pid_accuracy": 0.9831730723381042, "metrics/particle/class_2/efficiency": 0.3913043439388275, "metrics/particle/class_2/pid_accuracy": 0.8888888955116272, "metrics/particle/class_3/efficiency": 0.89682537317276, "metrics/particle/class_3/pid_accuracy": 0.991150438785553, "metrics/particle/class_4/efficiency": 0.970588207244873, "metrics/particle/class_4/pid_accuracy": 0.9494949579238892, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.01600000075995922, "metrics/particle/count/mae": 0.23199999332427979, "metrics/particle/count/prediction_mean": 4.599999904632568, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.04391258582472801, "metrics/particle/event/energy_response_mean": 0.9947448968887329, "metrics/particle/event/met_abs_error": 1.0496137142181396, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03878258168697357, "metrics/particle/event/scalar_pt_response_mean": 0.9933117628097534, "metrics/particle/event/vector_pt_closure": 0.04089183360338211, "metrics/particle/matched/delta_eta_abs_mean": 0.0071417889557778835, "metrics/particle/matched/delta_phi_abs_mean": 0.006468639709055424, "metrics/particle/matched/delta_r_mean": 0.010718229226768017, "metrics/particle/matched/energy_relative_abs_mean": 0.05963662266731262, "metrics/particle/matched/pid_accuracy": 0.9813084006309509, "metrics/particle/matched/pt_relative_abs_mean": 0.05994277074933052, "metrics/particle/matching/duplicate_fraction": 0.016521738842129707, "metrics/particle/matching/efficiency": 0.927209734916687, "metrics/particle/matching/f1": 0.9288194179534912, "metrics/particle/matching/purity": 0.9304347634315491, "metrics/particle/matching/target_coverage_dr0p05": 0.911611795425415, "metrics/particle/matching/target_coverage_dr0p10": 0.9462738037109375, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8596187233924866, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9367417693138123}, "step_train_time": 0.49208545684814453, "step_valid_time": 2.814079779998792, "step_total_time": 4395.369762182236, "elapsed_training_loop_seconds": 4395.369762182236, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0000428615580692, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02244146315461737, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 432, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 428, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9953596287703016, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9930555555555556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9942062572421785, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.006944444444444444, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9930394431554525, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9907407407407407, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9918887601390498, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9953596287703016, "step_test_time": 4.278685710998616, "step_plot_time": 3.4988521380000748, "step_validation_cycle_time": 10.924330742000166} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_38000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_38000.json new file mode 100644 index 0000000000000000000000000000000000000000..8f61a3a2d79a23187ea0926ac24687900687dc8f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_38000.json @@ -0,0 +1 @@ +{"epoch": 67.61565836298932, "train": {"Classification_binary": 0.011599414050579071, "Classification": 2.1623108636958932e-07, "Regression_pt": 0.02541092224419117, "Regression_eta": 0.00020138897525612265, "Regression_sin_phi": 0.00011518663814058527, "Regression_cos_phi": 0.0003183636872563511, "Regression_energy": 0.026544621214270592, "Total": 0.8592207431793213, "Total_unweighted": 0.06419011304078026}, "train_group": {}, "valid": {"Classification_binary": 2.79349422454834, "Classification": 0.018186476081609726, "Regression_pt": 0.0969085842370987, "Regression_eta": 0.01877191662788391, "Regression_sin_phi": 0.002497649984434247, "Regression_cos_phi": 0.0027752297464758158, "Regression_energy": 0.0998784750699997, "Total": 12.007200241088867, "Total_unweighted": 3.032512556295842, "metrics/particle/class_1/efficiency": 0.9721577763557434, "metrics/particle/class_1/pid_accuracy": 0.9856801629066467, "metrics/particle/class_2/efficiency": 0.3913043439388275, "metrics/particle/class_2/pid_accuracy": 0.8888888955116272, "metrics/particle/class_3/efficiency": 0.8789682388305664, "metrics/particle/class_3/pid_accuracy": 0.993228018283844, "metrics/particle/class_4/efficiency": 0.9509803652763367, "metrics/particle/class_4/pid_accuracy": 0.969072163105011, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.02800000086426735, "metrics/particle/count/mae": 0.2680000066757202, "metrics/particle/count/prediction_mean": 4.5879998207092285, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.04649835824966431, "metrics/particle/event/energy_response_mean": 1.0002552270889282, "metrics/particle/event/met_abs_error": 1.170737862586975, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04104306921362877, "metrics/particle/event/scalar_pt_response_mean": 0.9967734217643738, "metrics/particle/event/vector_pt_closure": 0.0427437424659729, "metrics/particle/matched/delta_eta_abs_mean": 0.007151692174375057, "metrics/particle/matched/delta_phi_abs_mean": 0.006601444445550442, "metrics/particle/matched/delta_r_mean": 0.010884268209338188, "metrics/particle/matched/energy_relative_abs_mean": 0.058670319616794586, "metrics/particle/matched/pid_accuracy": 0.9849340915679932, "metrics/particle/matched/pt_relative_abs_mean": 0.05889059603214264, "metrics/particle/matching/duplicate_fraction": 0.014821273274719715, "metrics/particle/matching/efficiency": 0.9202772974967957, "metrics/particle/matching/f1": 0.9230769276618958, "metrics/particle/matching/purity": 0.9258936643600464, "metrics/particle/matching/target_coverage_dr0p05": 0.909878671169281, "metrics/particle/matching/target_coverage_dr0p10": 0.9393414258956909, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8648180365562439, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9341421127319336}, "step_train_time": 0.48928236961364746, "step_valid_time": 2.8716636579993065, "step_total_time": 4514.7653028965, "elapsed_training_loop_seconds": 4514.7653028965, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0026375667966505, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.022138485525642904, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 428, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9930394431554525, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9976689976689976, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9953488372093023, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.002331002331002331, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9906759906759907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9930394431554525, "step_test_time": 4.293958285999906, "step_plot_time": 3.5687690879985894, "step_validation_cycle_time": 11.065777820000221} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_4000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_4000.json new file mode 100644 index 0000000000000000000000000000000000000000..291a438c6d2373c98478c14746c2c23fef986533 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_4000.json @@ -0,0 +1 @@ +{"epoch": 7.117437722419929, "train": {"Classification_binary": 0.8702100515365601, "Classification": 0.00992460735142231, "Regression_pt": 0.045693811029195786, "Regression_eta": 0.0007691332139074802, "Regression_sin_phi": 0.0013292781077325344, "Regression_cos_phi": 0.0009241621592082083, "Regression_energy": 0.04401800036430359, "Total": 4.079713344573975, "Total_unweighted": 0.97286904376233}, "train_group": {}, "valid": {"Classification_binary": 1.0222303867340088, "Classification": 0.007560228928923607, "Regression_pt": 0.09657656401395798, "Regression_eta": 0.012373578734695911, "Regression_sin_phi": 0.002492929808795452, "Regression_cos_phi": 0.0026334014255553484, "Regression_energy": 0.10340061038732529, "Total": 8.306849479675293, "Total_unweighted": 1.2472677000332624, "metrics/particle/class_1/efficiency": 0.9744779467582703, "metrics/particle/class_1/pid_accuracy": 0.9785714149475098, "metrics/particle/class_2/efficiency": 0.3913043439388275, "metrics/particle/class_2/pid_accuracy": 0.8888888955116272, "metrics/particle/class_3/efficiency": 0.8769841194152832, "metrics/particle/class_3/pid_accuracy": 0.9864253401756287, "metrics/particle/class_4/efficiency": 0.9607843160629272, "metrics/particle/class_4/pid_accuracy": 0.9591836929321289, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.978723406791687, "metrics/particle/count/bias_mean": -0.05999999865889549, "metrics/particle/count/mae": 0.19599999487400055, "metrics/particle/count/prediction_mean": 4.556000232696533, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.03813944011926651, "metrics/particle/event/energy_response_mean": 0.9896047711372375, "metrics/particle/event/met_abs_error": 1.036916971206665, "metrics/particle/event/scalar_pt_relative_abs_error": 0.036803197115659714, "metrics/particle/event/scalar_pt_response_mean": 0.9908962845802307, "metrics/particle/event/vector_pt_closure": 0.03924243152141571, "metrics/particle/matched/delta_eta_abs_mean": 0.007674661464989185, "metrics/particle/matched/delta_phi_abs_mean": 0.0064266137778759, "metrics/particle/matched/delta_r_mean": 0.011127176694571972, "metrics/particle/matched/energy_relative_abs_mean": 0.05798304080963135, "metrics/particle/matched/pid_accuracy": 0.9793038368225098, "metrics/particle/matched/pt_relative_abs_mean": 0.058455806225538254, "metrics/particle/matching/duplicate_fraction": 0.011413520202040672, "metrics/particle/matching/efficiency": 0.9211438298225403, "metrics/particle/matching/f1": 0.9271696209907532, "metrics/particle/matching/purity": 0.9332748055458069, "metrics/particle/matching/target_coverage_dr0p05": 0.9133448600769043, "metrics/particle/matching/target_coverage_dr0p10": 0.9445407390594482, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8665511012077332, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9358752369880676}, "step_train_time": 0.4719371795654297, "step_valid_time": 2.821571705999304, "step_total_time": 467.3461239337921, "elapsed_training_loop_seconds": 467.3461239337921, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9965410978273221, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.023307755706427846, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9930232558139535, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.991869918699187, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0069767441860465115, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9872241579558653, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.1647876240003825, "step_plot_time": 3.429326915999809, "step_validation_cycle_time": 10.726761249000447} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_5000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_5000.json new file mode 100644 index 0000000000000000000000000000000000000000..8f5a676887b3015e301db8421a1c8343f93a6999 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_5000.json @@ -0,0 +1 @@ +{"epoch": 8.896797153024911, "train": {"Classification_binary": 1.1555747985839844, "Classification": 0.0027325735427439213, "Regression_pt": 0.06219223141670227, "Regression_eta": 0.00260598654858768, "Regression_sin_phi": 8.43494271975942e-05, "Regression_cos_phi": 0.0004379276651889086, "Regression_energy": 0.05891942232847214, "Total": 3.502164602279663, "Total_unweighted": 1.2825472895128769}, "train_group": {}, "valid": {"Classification_binary": 1.0046310424804688, "Classification": 0.007810089271515608, "Regression_pt": 0.09455619752407074, "Regression_eta": 0.012277836911380291, "Regression_sin_phi": 0.002596735255792737, "Regression_cos_phi": 0.0024421783164143562, "Regression_energy": 0.10116644948720932, "Total": 8.24809455871582, "Total_unweighted": 1.2254805292468518, "metrics/particle/class_1/efficiency": 0.9675173759460449, "metrics/particle/class_1/pid_accuracy": 0.9880095720291138, "metrics/particle/class_2/efficiency": 0.3478260934352875, "metrics/particle/class_2/pid_accuracy": 0.875, "metrics/particle/class_3/efficiency": 0.8849206566810608, "metrics/particle/class_3/pid_accuracy": 0.9932735562324524, "metrics/particle/class_4/efficiency": 0.970588207244873, "metrics/particle/class_4/pid_accuracy": 0.9595959782600403, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.07599999755620956, "metrics/particle/count/mae": 0.2280000001192093, "metrics/particle/count/prediction_mean": 4.539999961853027, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.039106085896492004, "metrics/particle/event/energy_response_mean": 0.9862834215164185, "metrics/particle/event/met_abs_error": 1.0405313968658447, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03968629240989685, "metrics/particle/event/scalar_pt_response_mean": 0.9873424768447876, "metrics/particle/event/vector_pt_closure": 0.03958256170153618, "metrics/particle/matched/delta_eta_abs_mean": 0.0076528266072273254, "metrics/particle/matched/delta_phi_abs_mean": 0.0063772499561309814, "metrics/particle/matched/delta_r_mean": 0.01104694977402687, "metrics/particle/matched/energy_relative_abs_mean": 0.05822998657822609, "metrics/particle/matched/pid_accuracy": 0.9849624037742615, "metrics/particle/matched/pt_relative_abs_mean": 0.05907537043094635, "metrics/particle/matching/duplicate_fraction": 0.009691629558801651, "metrics/particle/matching/efficiency": 0.9220104217529297, "metrics/particle/matching/f1": 0.9296635985374451, "metrics/particle/matching/purity": 0.9374449253082275, "metrics/particle/matching/target_coverage_dr0p05": 0.9124783277511597, "metrics/particle/matching/target_coverage_dr0p10": 0.9393414258956909, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8648180365562439, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9367417693138123}, "step_train_time": 0.47607898712158203, "step_valid_time": 2.801170865000131, "step_total_time": 585.7146143913269, "elapsed_training_loop_seconds": 585.7146143913269, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.998756877742307, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.024559965475704937, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 428, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9930394431554525, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9930394431554525, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9930394431554525, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0069605568445475635, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9930394431554525, "step_test_time": 4.174753985999814, "step_plot_time": 3.425984825999876, "step_validation_cycle_time": 10.711853116999919} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_6000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_6000.json new file mode 100644 index 0000000000000000000000000000000000000000..dcae6de729abf3849c59d63d62bfe60352849c04 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_6000.json @@ -0,0 +1 @@ +{"epoch": 10.676156583629894, "train": {"Classification_binary": 0.6709461212158203, "Classification": 0.0009426476899534464, "Regression_pt": 0.08415216207504272, "Regression_eta": 0.000593104399740696, "Regression_sin_phi": 8.987523324321955e-05, "Regression_cos_phi": 0.0004903773660771549, "Regression_energy": 0.07578541338443756, "Total": 3.065387725830078, "Total_unweighted": 0.8329997013643151}, "train_group": {}, "valid": {"Classification_binary": 1.0385708808898926, "Classification": 0.007298212964087725, "Regression_pt": 0.09631999582052231, "Regression_eta": 0.012392891570925713, "Regression_sin_phi": 0.00264466623775661, "Regression_cos_phi": 0.002475786954164505, "Regression_energy": 0.10285703837871552, "Total": 8.333497047424316, "Total_unweighted": 1.262559472816065, "metrics/particle/class_1/efficiency": 0.9744779467582703, "metrics/particle/class_1/pid_accuracy": 0.9904761910438538, "metrics/particle/class_2/efficiency": 0.43478259444236755, "metrics/particle/class_2/pid_accuracy": 0.800000011920929, "metrics/particle/class_3/efficiency": 0.8829365372657776, "metrics/particle/class_3/pid_accuracy": 0.9955056309700012, "metrics/particle/class_4/efficiency": 0.9607843160629272, "metrics/particle/class_4/pid_accuracy": 0.9693877696990967, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.03999999910593033, "metrics/particle/count/mae": 0.20800000429153442, "metrics/particle/count/prediction_mean": 4.576000213623047, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.03844029828906059, "metrics/particle/event/energy_response_mean": 0.9889556765556335, "metrics/particle/event/met_abs_error": 1.0093321800231934, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03755743056535721, "metrics/particle/event/scalar_pt_response_mean": 0.9902475476264954, "metrics/particle/event/vector_pt_closure": 0.03817776218056679, "metrics/particle/matched/delta_eta_abs_mean": 0.007387831807136536, "metrics/particle/matched/delta_phi_abs_mean": 0.006251505110412836, "metrics/particle/matched/delta_r_mean": 0.010724524967372417, "metrics/particle/matched/energy_relative_abs_mean": 0.05813589319586754, "metrics/particle/matched/pid_accuracy": 0.9868791103363037, "metrics/particle/matched/pt_relative_abs_mean": 0.05890439823269844, "metrics/particle/matching/duplicate_fraction": 0.012237762100994587, "metrics/particle/matching/efficiency": 0.9246100783348083, "metrics/particle/matching/f1": 0.9286335706710815, "metrics/particle/matching/purity": 0.932692289352417, "metrics/particle/matching/target_coverage_dr0p05": 0.9185441732406616, "metrics/particle/matching/target_coverage_dr0p10": 0.9436741471290588, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8630849123001099, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9393414258956909}, "step_train_time": 0.4634110927581787, "step_valid_time": 2.8171582189997935, "step_total_time": 704.5458631515503, "elapsed_training_loop_seconds": 704.5458631515503, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9970109588251004, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.023386860437279333, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 428, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9930394431554525, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9930394431554525, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9930394431554525, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0069605568445475635, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9930394431554525, "step_test_time": 4.226006562000293, "step_plot_time": 3.483543856000324, "step_validation_cycle_time": 10.842541031999644} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_7000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_7000.json new file mode 100644 index 0000000000000000000000000000000000000000..82e39220dbb5e54d26da63e98148a964583bf1de --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_7000.json @@ -0,0 +1 @@ +{"epoch": 12.455516014234876, "train": {"Classification_binary": 1.1142284870147705, "Classification": 0.0030757500790059566, "Regression_pt": 0.09535934776067734, "Regression_eta": 0.026421234011650085, "Regression_sin_phi": 0.0060728564858436584, "Regression_cos_phi": 0.008340764790773392, "Regression_energy": 0.059591520577669144, "Total": 13.281160354614258, "Total_unweighted": 1.31308996072039}, "train_group": {}, "valid": {"Classification_binary": 1.0350755453109741, "Classification": 0.0078095025382936, "Regression_pt": 0.09711399674415588, "Regression_eta": 0.013088885694742203, "Regression_sin_phi": 0.0025173379108309746, "Regression_cos_phi": 0.002411339897662401, "Regression_energy": 0.10318765044212341, "Total": 8.400932312011719, "Total_unweighted": 1.2612042585387826, "metrics/particle/class_1/efficiency": 0.9675173759460449, "metrics/particle/class_1/pid_accuracy": 0.9760192036628723, "metrics/particle/class_2/efficiency": 0.3478260934352875, "metrics/particle/class_2/pid_accuracy": 0.875, "metrics/particle/class_3/efficiency": 0.8888888955116272, "metrics/particle/class_3/pid_accuracy": 0.9910714030265808, "metrics/particle/class_4/efficiency": 0.9509803652763367, "metrics/particle/class_4/pid_accuracy": 0.9793814420700073, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9680851101875305, "metrics/particle/count/bias_mean": -0.04399999976158142, "metrics/particle/count/mae": 0.20399999618530273, "metrics/particle/count/prediction_mean": 4.572000026702881, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.03742359206080437, "metrics/particle/event/energy_response_mean": 0.9937017560005188, "metrics/particle/event/met_abs_error": 1.008788824081421, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03842546418309212, "metrics/particle/event/scalar_pt_response_mean": 0.9928462505340576, "metrics/particle/event/vector_pt_closure": 0.038702789694070816, "metrics/particle/matched/delta_eta_abs_mean": 0.007238820195198059, "metrics/particle/matched/delta_phi_abs_mean": 0.006785096134990454, "metrics/particle/matched/delta_r_mean": 0.011125592514872551, "metrics/particle/matched/energy_relative_abs_mean": 0.057750023901462555, "metrics/particle/matched/pid_accuracy": 0.981203019618988, "metrics/particle/matched/pt_relative_abs_mean": 0.058163080364465714, "metrics/particle/matching/duplicate_fraction": 0.01312335953116417, "metrics/particle/matching/efficiency": 0.9220104217529297, "metrics/particle/matching/f1": 0.9264257550239563, "metrics/particle/matching/purity": 0.9308836460113525, "metrics/particle/matching/target_coverage_dr0p05": 0.9150779843330383, "metrics/particle/matching/target_coverage_dr0p10": 0.9428076148033142, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8674176931381226, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9384748935699463}, "step_train_time": 0.4679698944091797, "step_valid_time": 2.8291514960001223, "step_total_time": 822.8720171451569, "elapsed_training_loop_seconds": 822.8720171451569, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9997151694521202, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.022379575463989165, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.006993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9906759906759907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.225958752000224, "step_plot_time": 3.446976543999881, "step_validation_cycle_time": 10.812363342000026} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_8000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_8000.json new file mode 100644 index 0000000000000000000000000000000000000000..13c77ad6fa856df86d9123bbc15e867df767513f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_8000.json @@ -0,0 +1 @@ +{"epoch": 14.234875444839858, "train": {"Classification_binary": 0.7249996066093445, "Classification": 0.00021817749075125903, "Regression_pt": 0.016209635883569717, "Regression_eta": 0.0001505603431724012, "Regression_sin_phi": 5.799365317216143e-05, "Regression_cos_phi": 0.00014791643479838967, "Regression_energy": 0.015818027779459953, "Total": 1.2497378587722778, "Total_unweighted": 0.7576019181942684}, "train_group": {}, "valid": {"Classification_binary": 1.0413508415222168, "Classification": 0.00784123782068491, "Regression_pt": 0.08280453830957413, "Regression_eta": 0.012987528927624226, "Regression_sin_phi": 0.0026358915492892265, "Regression_cos_phi": 0.002531196689233184, "Regression_energy": 0.08936359733343124, "Total": 8.144262313842773, "Total_unweighted": 1.2395148321520537, "metrics/particle/class_1/efficiency": 0.9721577763557434, "metrics/particle/class_1/pid_accuracy": 0.9856801629066467, "metrics/particle/class_2/efficiency": 0.47826087474823, "metrics/particle/class_2/pid_accuracy": 0.8181818127632141, "metrics/particle/class_3/efficiency": 0.8809523582458496, "metrics/particle/class_3/pid_accuracy": 0.9932432174682617, "metrics/particle/class_4/efficiency": 0.9607843160629272, "metrics/particle/class_4/pid_accuracy": 0.9693877696990967, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.978723406791687, "metrics/particle/count/bias_mean": -0.07199999690055847, "metrics/particle/count/mae": 0.2240000069141388, "metrics/particle/count/prediction_mean": 4.544000148773193, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.037491925060749054, "metrics/particle/event/energy_response_mean": 0.9935764670372009, "metrics/particle/event/met_abs_error": 0.9619979858398438, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03861159458756447, "metrics/particle/event/scalar_pt_response_mean": 0.9931433200836182, "metrics/particle/event/vector_pt_closure": 0.03753633424639702, "metrics/particle/matched/delta_eta_abs_mean": 0.007348154205828905, "metrics/particle/matched/delta_phi_abs_mean": 0.0063894568011164665, "metrics/particle/matched/delta_r_mean": 0.010840174742043018, "metrics/particle/matched/energy_relative_abs_mean": 0.05745868757367134, "metrics/particle/matched/pid_accuracy": 0.9849905967712402, "metrics/particle/matched/pt_relative_abs_mean": 0.05787643417716026, "metrics/particle/matching/duplicate_fraction": 0.01056338008493185, "metrics/particle/matching/efficiency": 0.923743486404419, "metrics/particle/matching/f1": 0.9310043454170227, "metrics/particle/matching/purity": 0.9383803009986877, "metrics/particle/matching/target_coverage_dr0p05": 0.917677640914917, "metrics/particle/matching/target_coverage_dr0p10": 0.9436741471290588, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8648180365562439, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9367417693138123}, "step_train_time": 0.46846652030944824, "step_valid_time": 2.808855668000433, "step_total_time": 941.2834742069244, "elapsed_training_loop_seconds": 941.2834742069244, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0017200481964754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.025388409992777694, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907192575406032, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9953379953379954, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9930232558139535, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.004662004662004662, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9906759906759907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907192575406032, "step_test_time": 4.188327464999929, "step_plot_time": 3.4116563800007498, "step_validation_cycle_time": 10.786500848999822} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_9000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_9000.json new file mode 100644 index 0000000000000000000000000000000000000000..977e34dc8d33619f8e610b4c5ef5e033ec7a73a4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/history/step_9000.json @@ -0,0 +1 @@ +{"epoch": 16.01423487544484, "train": {"Classification_binary": 1.0993098020553589, "Classification": 0.0015386055456474423, "Regression_pt": 0.028200944885611534, "Regression_eta": 0.0003669544239528477, "Regression_sin_phi": 0.0001358583103865385, "Regression_cos_phi": 0.000403425598051399, "Regression_energy": 0.028278086334466934, "Total": 2.2076005935668945, "Total_unweighted": 1.1582336771534756}, "train_group": {}, "valid": {"Classification_binary": 1.0439940690994263, "Classification": 0.007883364334702492, "Regression_pt": 0.08828883618116379, "Regression_eta": 0.013532806187868118, "Regression_sin_phi": 0.0027303823735564947, "Regression_cos_phi": 0.0025088463444262743, "Regression_energy": 0.09472465515136719, "Total": 8.416863441467285, "Total_unweighted": 1.2536629596725106, "metrics/particle/class_1/efficiency": 0.9721577763557434, "metrics/particle/class_1/pid_accuracy": 0.9856801629066467, "metrics/particle/class_2/efficiency": 0.43478259444236755, "metrics/particle/class_2/pid_accuracy": 0.800000011920929, "metrics/particle/class_3/efficiency": 0.89682537317276, "metrics/particle/class_3/pid_accuracy": 0.9955752491950989, "metrics/particle/class_4/efficiency": 0.9803921580314636, "metrics/particle/class_4/pid_accuracy": 0.9700000286102295, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.978723406791687, "metrics/particle/count/bias_mean": -0.03200000151991844, "metrics/particle/count/mae": 0.2240000069141388, "metrics/particle/count/prediction_mean": 4.584000110626221, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.03963868319988251, "metrics/particle/event/energy_response_mean": 0.9900044202804565, "metrics/particle/event/met_abs_error": 1.0601307153701782, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03995322808623314, "metrics/particle/event/scalar_pt_response_mean": 0.9890037775039673, "metrics/particle/event/vector_pt_closure": 0.04016711562871933, "metrics/particle/matched/delta_eta_abs_mean": 0.007265086285769939, "metrics/particle/matched/delta_phi_abs_mean": 0.007148122880607843, "metrics/particle/matched/delta_r_mean": 0.011354204267263412, "metrics/particle/matched/energy_relative_abs_mean": 0.05883600562810898, "metrics/particle/matched/pid_accuracy": 0.9860464930534363, "metrics/particle/matched/pt_relative_abs_mean": 0.05969316512346268, "metrics/particle/matching/duplicate_fraction": 0.010471204295754433, "metrics/particle/matching/efficiency": 0.9315424561500549, "metrics/particle/matching/f1": 0.9347826242446899, "metrics/particle/matching/purity": 0.9380453824996948, "metrics/particle/matching/target_coverage_dr0p05": 0.917677640914917, "metrics/particle/matching/target_coverage_dr0p10": 0.9488734602928162, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8708838820457458, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9436741471290588}, "step_train_time": 0.4827463626861572, "step_valid_time": 2.831482153000252, "step_total_time": 1059.194934129715, "elapsed_training_loop_seconds": 1059.194934129715, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9989799410277562, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.025263094390659768, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.988399071925754, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9906976744186047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.006993006993006993, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9860788863109049, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9906759906759907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.988399071925754, "step_test_time": 4.24750354599928, "step_plot_time": 3.5059136829995623, "step_validation_cycle_time": 10.920811754999704} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..abb0b2d0981df63f67ac8dfd1f42084a2ec67802 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_4mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 30, "checkpoint_freq": 2000, "val_freq": 1000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", "relaxed_load": false, "load_weights_only": true, "reset_task_loss_weights": true, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_z_tautau_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a2aaab90b4f0eea42f5e52779ec0f04aaef68436 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c804cccd2aa3df458fbb94817f4f7f7c18eca477514190f90493086398dca01 +size 4506 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..bcdfa6a4e5d0e5449b42aef160bd6989be8f2143 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..8e8d90d9738de428a44991d037793e7df2f65cb8 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..9a8abce56170eefeccb9054cdd7091b617465306 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..23f0c0cd591ebec21e6bc84551a4df94175ac11b Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..67824b63070d7c257633900534c7697ac7a1ffff Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..b17a7bb04d76a092168915b28feb15cf1d949fe6 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0a68ee554dde686ab76a5a7b6f22d8e13ab7f0dc Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..5886a7b70adf95aacbabd1d7f399d9c06311731e Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..146d391b1377c164ae9e0023d89fb44133bed3d3 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..0553374a140b6ac20e5b37129c028d8034d35447 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..36e0d0c135201723176d62a6e1d10b1f3e5735c3 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..8a0c567fa8c69e818cf505c64fe519b40b02c8ad Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/plots_step_38000/cld_edm_z_tautau_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/runs/train/events.out.tfevents.1794731515.workergpu169.1647342.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/runs/train/events.out.tfevents.1794731515.workergpu169.1647342.0 new file mode 100644 index 0000000000000000000000000000000000000000..49945c19f928258f866f1dfc5116e75338d85af9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/runs/train/events.out.tfevents.1794731515.workergpu169.1647342.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86bf198f815d05213b163d55881d3e6c33eb262155652db41d883872ae63a2f8 +size 1399333 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/runs/valid/events.out.tfevents.1794731515.workergpu169.1647342.1 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/runs/valid/events.out.tfevents.1794731515.workergpu169.1647342.1 new file mode 100644 index 0000000000000000000000000000000000000000..6ef2d15cb050a04a2352d5d4e68c3ae7aa365394 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/runs/valid/events.out.tfevents.1794731515.workergpu169.1647342.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:174cdfed1b3fa27694080d37fb4e2f3ffbb1461f825dec3c7a3d9220c45a7711 +size 17541820 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..e3950fd51d33904dab111bd534e7e9f0659831bb --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/scenario-manifest.json @@ -0,0 +1,1280 @@ +{ + "scenario": { + "name": "pf_fm_finetuning", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all" + }, + "allowed_variant_differences": [ + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets", + "data_config" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_finetuning", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_ecal_4mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_4mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all", + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_4mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 30, + "checkpoint_freq": 2000, + "val_freq": 1000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "relaxed_load": false, + "load_weights_only": true, + "reset_task_loss_weights": true, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_z_tautau_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_4mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--load", + "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "1000", + "--checkpoint_freq", + "2000", + "--patience", + "30", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--load_weights_only", + "true", + "--reset_task_loss_weights", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.feature_bag", + "true", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.trainable", + "all", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9042a4d6dfecd7a5b73ed5384f6fcfa3172f8912 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_4mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_z_tautau_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +load_weights_only: true +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 30 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: true +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_z_tautau_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 1000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/train.log b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f8d305f0f5bb31c9ca6387009efe6ce0bcdba9f6 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/train.log @@ -0,0 +1,2966 @@ +rank0 [2026-11-15 03:31:52,803] INFO : Will use single-gpu: NVIDIA H100 PCIe +rank0 [2026-11-15 03:31:52,803] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-15 03:31:53,265] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 03:31:53,265] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 03:31:53,266] INFO : Instantiating model +rank0 [2026-11-15 03:31:53,290] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 03:31:53,290] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 03:31:53,292] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 03:31:53,292] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 03:31:53,292] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 03:31:53,292] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 03:31:53,292] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:31:53,294] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 03:31:53,294] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:31:53,297] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 03:31:53,297] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:31:53,300] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 03:31:53,300] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:31:53,303] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 03:31:53,303] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:31:53,305] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 03:31:53,305] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:31:53,308] INFO : Backbone initialization took 0.02s +rank0 [2026-11-15 03:31:53,308] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 03:31:53,308] INFO : Initializing output DNNs +rank0 [2026-11-15 03:31:53,309] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:31:53,312] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:31:53,319] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-15 03:31:53,319] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 03:31:53,319] INFO : jagged_attention enabled=True +rank0 [2026-11-15 03:31:53,319] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 03:31:53,320] INFO : task_queries enabled=True +rank0 [2026-11-15 03:31:53,320] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 03:31:53,320] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 03:31:53,320] INFO : classification_query parameters: 256 +rank0 [2026-11-15 03:31:53,320] INFO : regression_query parameters: 256 +rank0 [2026-11-15 03:31:53,320] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 03:31:53,320] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 03:31:53,320] INFO : output_mode=elementwise +rank0 [2026-11-15 03:31:53,320] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 03:31:53,320] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 03:31:53,320] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 03:31:53,320] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 03:31:53,320] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 03:31:53,320] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 03:31:53,320] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 03:31:53,320] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 03:31:53,321] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 03:31:53,321] INFO : MLPF __init__ done +rank0 [2026-11-15 03:31:53,322] INFO : Instantiated model +rank0 [2026-11-15 03:31:53,322] INFO : Moving model to device rank=0 +rank0 [2026-11-15 03:31:53,330] INFO : Moved model to device rank=0 +rank0 [2026-11-15 03:31:53,331] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 03:31:53,449] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 03:31:53,449] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 03:31:53,449] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 03:31:53,543] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 03:31:53,543] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 03:31:53,544] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 03:31:53,544] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 03:31:53,544] INFO : Total parameters: 3910930 +rank0 [2026-11-15 03:31:53,547] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 03:31:53,552] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062 +rank0 [2026-11-15 03:31:53,552] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062 +rank0 [2026-11-15 03:31:53,552] INFO : Getting dataloaders +rank0 [2026-11-15 03:31:53,552] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:31:55,022] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:31:55,093] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 03:31:55,155] INFO : train_dataset: cld_edm_z_tautau_pf, 898 +rank0 [2026-11-15 03:31:55,197] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:31:55,243] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 03:31:55,291] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:31:55,349] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:31:55,418] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:31:55,651] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 03:31:55,695] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:31:55,695] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 03:31:55,695] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 03:31:55,696] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-15 03:31:55,696] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:31:55,707] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,719] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,730] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,741] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,752] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,762] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,772] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,783] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,793] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,803] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,803] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 03:31:55,803] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 03:31:55,851] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:31:55,976] INFO : Got dataloaders +rank0 [2026-11-15 03:31:55,976] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-15 03:31:55,976] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-15 03:31:55,976] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 03:31:55,986] INFO : Creating train_iterator +rank0 [2026-11-15 03:31:55,986] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:31:56,010] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:31:56,134] INFO : Created train_iterator +rank0 [2026-11-15 03:32:08,426] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.4839125296473503, 'Classification': 0.016125061931088567, 'Regression_pt': 0.11590320996940136, 'Regression_eta': 0.010036410794418771, 'Regression_sin_phi': 0.0025469883670302808, 'Regression_cos_phi': 0.004999627624201821, 'Regression_energy': 0.11904318055137991}, weights={'Classification_binary': 1.0, 'Classification': 92.02523040771484, 'Regression_pt': 12.803031921386719, 'Regression_eta': 147.8529052734375, 'Regression_sin_phi': 582.6145629882812, 'Regression_cos_phi': 296.8045959472656, 'Regression_energy': 12.465330123901367} +rank0 [2026-11-15 03:32:58,579] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 03:32:58,579] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:33:45,886] INFO : Step 1000/500000 rank0 | Train Loss: 4.5382 | Train Loss Unweighted: 1.0758 | Train Loss Components: Classification: 0.0109 | Classification_binary: 0.8709 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0969 | Regression_eta: 0.0003 | Regression_pt: 0.0963 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 1.0033 | Classification_binary: 0.8709 | Regression_cos_phi: 0.0872 | Regression_energy: 1.2082 | Regression_eta: 0.0479 | Regression_pt: 1.2329 | Regression_sin_phi: 0.0879 | LR: 2.01e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4737s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0379 | max_optimizer_step: 0.0575 | max_tensorboard_logging: 0.1365 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2604.2 | uss_MB: 1369.6 | child_rss_MB: 22943.0 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:33:45,887] INFO : Running validation on rank0 +rank0 [2026-11-15 03:33:45,887] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:33:45,888] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:33:46,057] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 03:33:46,233] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:33:46,301] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:33:46,354] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:33:46,408] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:33:46,454] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:33:46,507] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:33:46,562] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:33:46,609] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:33:46,688] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:33:46,762] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:33:46,813] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:33:46,867] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:33:46,912] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:33:46,966] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:33:47,019] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:33:47,067] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:33:47,071] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:33:47,073] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:33:47,075] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:33:47,077] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:33:47,079] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:33:47,080] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:33:47,082] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:33:47,084] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:33:47,084] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:33:47,086] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:33:47,088] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:33:47,089] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:33:47,091] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:33:47,093] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:33:47,095] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:33:48,571] INFO : VALIDATION | Step=1000/500000 | Train Loss=4.5382 | Valid Loss=8.9650 | Train Loss Unweighted=1.0758 | Valid Loss Unweighted=1.2835 | Stale=0 | ETA=936.3m +rank0 [2026-11-15 03:33:48,571] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:33:48,646] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:33:48,654] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:33:48,695] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:33:49,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:33:49,565] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:33:49,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:33:50,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:33:50,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:33:50,523] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:33:50,756] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:33:50,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:33:51,221] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:33:51,456] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:33:51,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:33:51,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:33:52,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:33:52,415] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:33:52,653] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:33:52,869] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:33:52,930] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:33:52,939] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:33:53,464] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:33:54,254] INFO : Plotted number of elements +rank0 [2026-11-15 03:33:55,465] INFO : Plotted jets +rank0 [2026-11-15 03:33:56,790] INFO : Plotted jet ratio +rank0 [2026-11-15 03:34:10,470] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 03:34:10,470] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:35:10,308] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 03:35:10,308] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:35:44,405] INFO : Step 2000/500000 rank0 | Train Loss: 3.1057 | Train Loss Unweighted: 1.0979 | Train Loss Components: Classification: 0.0031 | Classification_binary: 1.0078 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0422 | Regression_eta: 0.0003 | Regression_pt: 0.0429 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.2882 | Classification_binary: 1.0078 | Regression_cos_phi: 0.2440 | Regression_energy: 0.5255 | Regression_eta: 0.0429 | Regression_pt: 0.5499 | Regression_sin_phi: 0.4475 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4780s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0381 | max_optimizer_step: 0.0604 | max_tensorboard_logging: 0.1394 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2727.3 | uss_MB: 1500.1 | child_rss_MB: 22739.5 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:35:44,407] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 03:35:44,475] INFO : Running validation on rank0 +rank0 [2026-11-15 03:35:44,475] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:35:44,475] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:35:44,594] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 03:35:44,698] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:35:44,796] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:35:44,863] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:35:44,927] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:35:44,990] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:35:45,054] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:35:45,119] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:35:45,224] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:35:45,331] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:35:45,438] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:35:45,505] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:35:45,571] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:35:45,635] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:35:45,700] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:35:45,766] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:35:45,862] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:35:45,868] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:35:45,870] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:35:45,872] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:35:45,874] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:35:45,876] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:35:45,877] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:35:45,879] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:35:45,881] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:35:45,881] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:35:45,883] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:35:45,885] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:35:45,886] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:35:45,888] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:35:45,890] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:35:45,892] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:35:47,367] INFO : VALIDATION | Step=2000/500000 | Train Loss=3.1057 | Valid Loss=8.2246 | Train Loss Unweighted=1.0979 | Valid Loss Unweighted=1.2467 | Stale=0 | ETA=960.2m +rank0 [2026-11-15 03:35:47,368] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:35:47,440] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:35:47,452] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:35:47,501] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:35:47,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:35:48,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:35:48,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:35:48,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:35:48,918] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:35:49,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:35:49,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:35:49,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:35:49,859] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:35:50,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:35:50,329] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:35:50,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:35:50,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:35:51,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:35:51,276] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:35:51,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:35:51,552] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:35:51,557] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:35:52,074] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:35:52,486] INFO : Plotted number of elements +rank0 [2026-11-15 03:35:53,632] INFO : Plotted jets +rank0 [2026-11-15 03:35:54,953] INFO : Plotted jet ratio +rank0 [2026-11-15 03:36:21,794] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 03:36:21,794] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:37:21,794] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 03:37:21,794] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:37:42,551] INFO : Step 3000/500000 rank0 | Train Loss: 2.7435 | Train Loss Unweighted: 0.6196 | Train Loss Components: Classification: 0.0054 | Classification_binary: 0.5119 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0464 | Regression_eta: 0.0014 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.4994 | Classification_binary: 0.5119 | Regression_cos_phi: 0.1296 | Regression_energy: 0.5790 | Regression_eta: 0.2120 | Regression_pt: 0.6875 | Regression_sin_phi: 0.1240 | LR: 2.05e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4624s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0379 | max_optimizer_step: 0.0591 | max_tensorboard_logging: 0.1277 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2780.4 | uss_MB: 1553.3 | child_rss_MB: 22739.7 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:37:42,553] INFO : Running validation on rank0 +rank0 [2026-11-15 03:37:42,553] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:37:42,553] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:37:42,671] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 03:37:42,775] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:37:42,873] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:37:42,938] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:37:43,002] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:37:43,066] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:37:43,132] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:37:43,197] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:37:43,304] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:37:43,413] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:37:43,523] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:37:43,591] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:37:43,656] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:37:43,718] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:37:43,782] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:37:43,845] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:37:43,938] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:37:43,944] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:37:43,946] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:37:43,948] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:37:43,950] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:37:43,952] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:37:43,953] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:37:43,955] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:37:43,957] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:37:43,957] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:37:43,959] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:37:43,961] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:37:43,962] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:37:43,964] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:37:43,966] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:37:43,968] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:37:45,422] INFO : VALIDATION | Step=3000/500000 | Train Loss=2.7435 | Valid Loss=8.2064 | Train Loss Unweighted=0.6196 | Valid Loss Unweighted=1.2760 | Stale=0 | ETA=964.8m +rank0 [2026-11-15 03:37:45,422] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:37:45,496] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:37:45,503] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:37:45,549] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:37:45,994] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:37:46,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:37:46,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:37:46,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:37:46,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:37:47,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:37:47,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:37:47,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:37:47,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:37:48,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:37:48,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:37:48,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:37:48,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:37:49,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:37:49,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:37:49,533] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:37:49,586] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:37:49,592] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:37:50,133] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:37:50,545] INFO : Plotted number of elements +rank0 [2026-11-15 03:37:51,698] INFO : Plotted jets +rank0 [2026-11-15 03:37:53,016] INFO : Plotted jet ratio +rank0 [2026-11-15 03:38:33,010] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 03:38:33,011] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:39:32,965] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 03:39:32,965] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:39:40,434] INFO : Step 4000/500000 rank0 | Train Loss: 4.0797 | Train Loss Unweighted: 0.9729 | Train Loss Components: Classification: 0.0099 | Classification_binary: 0.8702 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0440 | Regression_eta: 0.0008 | Regression_pt: 0.0457 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.9133 | Classification_binary: 0.8702 | Regression_cos_phi: 0.2743 | Regression_energy: 0.5487 | Regression_eta: 0.1137 | Regression_pt: 0.5850 | Regression_sin_phi: 0.7745 | LR: 2.08e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4711s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0377 | max_loss: 0.0027 | max_backward: 0.0383 | max_optimizer_step: 0.0603 | max_tensorboard_logging: 0.1316 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2811.4 | uss_MB: 1584.4 | child_rss_MB: 22739.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:39:40,435] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 03:39:40,509] INFO : Running validation on rank0 +rank0 [2026-11-15 03:39:40,509] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:39:40,509] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:39:40,629] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 03:39:40,736] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:39:40,835] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:39:40,900] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:39:40,964] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:39:41,026] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:39:41,091] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:39:41,156] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:39:41,266] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:39:41,380] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:39:41,492] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:39:41,563] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:39:41,629] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:39:41,694] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:39:41,758] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:39:41,822] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:39:41,915] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:39:41,921] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:39:41,923] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:39:41,924] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:39:41,927] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:39:41,928] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:39:41,929] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:39:41,931] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:39:41,933] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:39:41,934] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:39:41,936] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:39:41,936] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:39:41,938] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:39:41,940] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:39:41,942] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:39:41,944] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:39:43,334] INFO : VALIDATION | Step=4000/500000 | Train Loss=4.0797 | Valid Loss=8.3068 | Train Loss Unweighted=0.9729 | Valid Loss Unweighted=1.2473 | Stale=1 | ETA=965.9m +rank0 [2026-11-15 03:39:43,334] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:39:43,409] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:39:43,417] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:39:43,461] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:39:43,898] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:39:44,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:39:44,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:39:44,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:39:44,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:39:45,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:39:45,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:39:45,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:39:45,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:39:46,027] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:39:46,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:39:46,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:39:46,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:39:46,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:39:47,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:39:47,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:39:47,497] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:39:47,502] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:39:48,030] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:39:48,449] INFO : Plotted number of elements +rank0 [2026-11-15 03:39:49,595] INFO : Plotted jets +rank0 [2026-11-15 03:39:50,921] INFO : Plotted jet ratio +rank0 [2026-11-15 03:40:44,231] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 03:40:44,231] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:41:38,896] INFO : Step 5000/500000 rank0 | Train Loss: 3.5022 | Train Loss Unweighted: 1.2825 | Train Loss Components: Classification: 0.0027 | Classification_binary: 1.1556 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0589 | Regression_eta: 0.0026 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.2515 | Classification_binary: 1.1556 | Regression_cos_phi: 0.1300 | Regression_energy: 0.7345 | Regression_eta: 0.3853 | Regression_pt: 0.7962 | Regression_sin_phi: 0.0491 | LR: 2.13e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4755s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0408 | max_loss: 0.0028 | max_backward: 0.0398 | max_optimizer_step: 0.0605 | max_tensorboard_logging: 0.1278 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2842.5 | uss_MB: 1615.4 | child_rss_MB: 22739.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:41:38,897] INFO : Running validation on rank0 +rank0 [2026-11-15 03:41:38,897] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:41:38,898] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:41:39,015] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 03:41:39,117] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:41:39,216] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:41:39,282] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:41:39,345] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:41:39,409] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:41:39,474] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:41:39,538] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:41:39,646] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:41:39,753] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:41:39,862] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:41:39,929] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:41:39,994] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:41:40,059] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:41:40,125] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:41:40,191] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:41:40,286] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:41:40,292] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:41:40,294] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:41:40,297] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:41:40,299] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:41:40,301] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:41:40,301] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:41:40,303] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:41:40,305] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:41:40,306] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:41:40,308] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:41:40,310] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:41:40,310] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:41:40,312] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:41:40,314] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:41:40,316] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:41:41,702] INFO : VALIDATION | Step=5000/500000 | Train Loss=3.5022 | Valid Loss=8.2481 | Train Loss Unweighted=1.2825 | Valid Loss Unweighted=1.2255 | Stale=2 | ETA=966.4m +rank0 [2026-11-15 03:41:41,702] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:41:41,774] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:41:41,781] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:41:41,826] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:41:42,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:41:42,500] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:41:42,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:41:42,979] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:41:43,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:41:43,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:41:43,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:41:43,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:41:44,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:41:44,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:41:44,646] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:41:44,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:41:45,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:41:45,368] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:41:45,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:41:45,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:41:45,875] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:41:45,881] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:41:46,402] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:41:46,818] INFO : Plotted number of elements +rank0 [2026-11-15 03:41:47,965] INFO : Plotted jets +rank0 [2026-11-15 03:41:49,297] INFO : Plotted jet ratio +rank0 [2026-11-15 03:41:55,828] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 03:41:55,828] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:42:56,497] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 03:42:56,497] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:43:37,641] INFO : Step 6000/500000 rank0 | Train Loss: 3.0654 | Train Loss Unweighted: 0.8330 | Train Loss Components: Classification: 0.0009 | Classification_binary: 0.6709 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0758 | Regression_eta: 0.0006 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0867 | Classification_binary: 0.6709 | Regression_cos_phi: 0.1455 | Regression_energy: 0.9447 | Regression_eta: 0.0877 | Regression_pt: 1.0774 | Regression_sin_phi: 0.0524 | LR: 2.19e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4628s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0344 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0594 | max_tensorboard_logging: 0.1268 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2861.6 | uss_MB: 1641.9 | child_rss_MB: 22737.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:43:37,644] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 03:43:37,713] INFO : Running validation on rank0 +rank0 [2026-11-15 03:43:37,713] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:43:37,713] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:43:37,835] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 03:43:37,941] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:43:38,040] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:43:38,107] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:43:38,173] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:43:38,238] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:43:38,304] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:43:38,370] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:43:38,479] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:43:38,588] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:43:38,696] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:43:38,763] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:43:38,827] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:43:38,890] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:43:38,954] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:43:39,018] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:43:39,113] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:43:39,118] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:43:39,121] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:43:39,123] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:43:39,125] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:43:39,127] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:43:39,127] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:43:39,129] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:43:39,131] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:43:39,132] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:43:39,133] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:43:39,135] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:43:39,136] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:43:39,138] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:43:39,140] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:43:39,142] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:43:40,539] INFO : VALIDATION | Step=6000/500000 | Train Loss=3.0654 | Valid Loss=8.3335 | Train Loss Unweighted=0.8330 | Valid Loss Unweighted=1.2626 | Stale=3 | ETA=966.8m +rank0 [2026-11-15 03:43:40,539] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:43:40,612] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:43:40,620] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:43:40,671] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:43:41,129] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:43:41,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:43:41,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:43:41,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:43:42,096] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:43:42,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:43:42,578] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:43:42,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:43:43,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:43:43,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:43:43,539] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:43:43,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:43:44,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:43:44,258] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:43:44,492] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:43:44,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:43:44,763] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:43:44,771] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:43:45,307] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:43:45,719] INFO : Plotted number of elements +rank0 [2026-11-15 03:43:46,884] INFO : Plotted jets +rank0 [2026-11-15 03:43:48,240] INFO : Plotted jet ratio +rank0 [2026-11-15 03:44:08,037] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 03:44:08,037] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:45:08,267] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 03:45:08,267] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:45:36,026] INFO : Step 7000/500000 rank0 | Train Loss: 13.2812 | Train Loss Unweighted: 1.3131 | Train Loss Components: Classification: 0.0031 | Classification_binary: 1.1142 | Regression_cos_phi: 0.0083 | Regression_energy: 0.0596 | Regression_eta: 0.0264 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.2830 | Classification_binary: 1.1142 | Regression_cos_phi: 2.4756 | Regression_energy: 0.7428 | Regression_eta: 3.9065 | Regression_pt: 1.2209 | Regression_sin_phi: 3.5381 | LR: 2.26e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4674s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0389 | max_optimizer_step: 0.0605 | max_tensorboard_logging: 0.1263 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2881.0 | uss_MB: 1654.0 | child_rss_MB: 22736.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:45:36,027] INFO : Running validation on rank0 +rank0 [2026-11-15 03:45:36,027] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:45:36,027] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:45:36,147] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 03:45:36,249] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:45:36,348] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:45:36,416] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:45:36,483] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:45:36,550] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:45:36,618] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:45:36,685] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:45:36,799] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:45:36,909] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:45:37,019] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:45:37,085] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:45:37,149] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:45:37,212] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:45:37,277] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:45:37,342] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:45:37,440] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:45:37,445] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:45:37,448] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:45:37,450] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:45:37,452] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:45:37,454] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:45:37,455] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:45:37,455] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:45:37,457] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:45:37,459] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:45:37,460] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:45:37,463] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:45:37,463] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:45:37,465] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:45:37,467] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:45:37,468] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:45:38,861] INFO : VALIDATION | Step=7000/500000 | Train Loss=13.2812 | Valid Loss=8.4009 | Train Loss Unweighted=1.3131 | Valid Loss Unweighted=1.2612 | Stale=4 | ETA=965.9m +rank0 [2026-11-15 03:45:38,861] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:45:38,934] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:45:38,940] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:45:38,985] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:45:39,478] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:45:39,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:45:39,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:45:40,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:45:40,446] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:45:40,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:45:40,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:45:41,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:45:41,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:45:41,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:45:41,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:45:42,113] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:45:42,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:45:42,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:45:42,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:45:43,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:45:43,085] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:45:43,091] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:45:43,624] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:45:44,037] INFO : Plotted number of elements +rank0 [2026-11-15 03:45:45,193] INFO : Plotted jets +rank0 [2026-11-15 03:45:46,527] INFO : Plotted jet ratio +rank0 [2026-11-15 03:46:19,614] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 03:46:19,614] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:47:19,951] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 03:47:19,951] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:47:34,390] INFO : Step 8000/500000 rank0 | Train Loss: 1.2497 | Train Loss Unweighted: 0.7576 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.7250 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0158 | Regression_eta: 0.0002 | Regression_pt: 0.0162 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0201 | Classification_binary: 0.7250 | Regression_cos_phi: 0.0439 | Regression_energy: 0.1972 | Regression_eta: 0.0223 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0338 | LR: 2.34e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4678s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0344 | max_loss: 0.0025 | max_backward: 0.0380 | max_optimizer_step: 0.0594 | max_tensorboard_logging: 0.1306 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2908.0 | uss_MB: 1681.1 | child_rss_MB: 22735.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:47:34,392] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 03:47:34,459] INFO : Running validation on rank0 +rank0 [2026-11-15 03:47:34,459] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:47:34,459] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:47:34,576] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 03:47:34,680] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:47:34,781] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:47:34,849] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:47:34,914] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:47:34,979] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:47:35,046] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:47:35,113] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:47:35,222] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:47:35,330] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:47:35,438] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:47:35,504] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:47:35,568] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:47:35,631] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:47:35,694] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:47:35,758] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:47:35,854] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:47:35,860] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:47:35,862] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:47:35,864] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:47:35,866] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:47:35,868] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:47:35,869] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:47:35,871] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:47:35,871] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:47:35,873] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:47:35,875] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:47:35,877] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:47:35,877] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:47:35,879] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:47:35,881] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:47:35,882] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:47:37,340] INFO : VALIDATION | Step=8000/500000 | Train Loss=1.2497 | Valid Loss=8.1443 | Train Loss Unweighted=0.7576 | Valid Loss Unweighted=1.2395 | Stale=0 | ETA=964.9m +rank0 [2026-11-15 03:47:37,340] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:47:37,414] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:47:37,419] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:47:37,465] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:47:37,929] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:47:38,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:47:38,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:47:38,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:47:38,878] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:47:39,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:47:39,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:47:39,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:47:39,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:47:40,077] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:47:40,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:47:40,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:47:40,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:47:41,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:47:41,251] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:47:41,473] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:47:41,527] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:47:41,532] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:47:42,058] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:47:42,463] INFO : Plotted number of elements +rank0 [2026-11-15 03:47:43,616] INFO : Plotted jets +rank0 [2026-11-15 03:47:44,931] INFO : Plotted jet ratio +rank0 [2026-11-15 03:48:31,145] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 03:48:31,145] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:49:31,115] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 03:49:31,115] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:49:32,346] INFO : Step 9000/500000 rank0 | Train Loss: 2.2076 | Train Loss Unweighted: 1.1582 | Train Loss Components: Classification: 0.0015 | Classification_binary: 1.0993 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0283 | Regression_eta: 0.0004 | Regression_pt: 0.0282 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.1416 | Classification_binary: 1.0993 | Regression_cos_phi: 0.1197 | Regression_energy: 0.3525 | Regression_eta: 0.0543 | Regression_pt: 0.3611 | Regression_sin_phi: 0.0792 | LR: 2.43e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4825s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0373 | max_loss: 0.0027 | max_backward: 0.0396 | max_optimizer_step: 0.0610 | max_tensorboard_logging: 0.1310 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2862.3 | uss_MB: 1635.4 | child_rss_MB: 22739.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:49:32,348] INFO : Running validation on rank0 +rank0 [2026-11-15 03:49:32,348] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:49:32,348] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:49:32,468] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 03:49:32,574] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:49:32,673] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:49:32,740] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:49:32,805] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:49:32,869] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:49:32,934] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:49:32,999] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:49:33,107] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:49:33,215] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:49:33,325] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:49:33,393] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:49:33,458] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:49:33,524] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:49:33,590] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:49:33,656] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:49:33,755] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:49:33,761] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:49:33,763] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:49:33,765] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:49:33,767] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:49:33,769] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:49:33,770] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:49:33,772] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:49:33,774] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:49:33,774] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:49:33,775] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:49:33,778] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:49:33,779] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:49:33,781] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:49:33,783] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:49:33,785] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:49:35,184] INFO : VALIDATION | Step=9000/500000 | Train Loss=2.2076 | Valid Loss=8.4169 | Train Loss Unweighted=1.1582 | Valid Loss Unweighted=1.2537 | Stale=1 | ETA=963.1m +rank0 [2026-11-15 03:49:35,184] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:49:35,257] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:49:35,260] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:49:35,306] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:49:35,749] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:49:36,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:49:36,256] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:49:36,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:49:36,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:49:36,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:49:37,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:49:37,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:49:37,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:49:37,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:49:38,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:49:38,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:49:38,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:49:38,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:49:39,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:49:39,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:49:39,430] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:49:39,436] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:49:39,968] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:49:40,389] INFO : Plotted number of elements +rank0 [2026-11-15 03:49:41,541] INFO : Plotted jets +rank0 [2026-11-15 03:49:42,928] INFO : Plotted jet ratio +rank0 [2026-11-15 03:50:42,854] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 03:50:42,854] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:51:31,199] INFO : Step 10000/500000 rank0 | Train Loss: 1.2373 | Train Loss Unweighted: 0.4699 | Train Loss Components: Classification: 0.0023 | Classification_binary: 0.4373 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0144 | Regression_eta: 0.0006 | Regression_pt: 0.0150 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.2080 | Classification_binary: 0.4373 | Regression_cos_phi: 0.0417 | Regression_energy: 0.1800 | Regression_eta: 0.0899 | Regression_pt: 0.1918 | Regression_sin_phi: 0.0886 | LR: 2.52e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4722s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0375 | max_loss: 0.0028 | max_backward: 0.0400 | max_optimizer_step: 0.0640 | max_tensorboard_logging: 0.1285 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2965.3 | uss_MB: 1738.4 | child_rss_MB: 22741.0 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:51:31,201] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 03:51:31,268] INFO : Running validation on rank0 +rank0 [2026-11-15 03:51:31,268] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:51:31,268] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:51:31,387] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 03:51:31,489] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:51:31,588] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:51:31,655] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:51:31,719] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:51:31,783] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:51:31,848] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:51:31,915] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:51:32,026] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:51:32,142] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:51:32,256] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:51:32,326] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:51:32,392] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:51:32,457] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:51:32,522] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:51:32,587] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:51:32,684] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:51:32,690] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:51:32,692] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:51:32,695] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:51:32,696] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:51:32,697] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:51:32,699] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:51:32,701] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:51:32,703] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:51:32,703] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:51:32,705] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:51:32,707] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:51:32,708] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:51:32,710] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:51:32,711] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:51:32,714] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:51:34,111] INFO : VALIDATION | Step=10000/500000 | Train Loss=1.2373 | Valid Loss=8.2623 | Train Loss Unweighted=0.4699 | Valid Loss Unweighted=1.2536 | Stale=2 | ETA=962.1m +rank0 [2026-11-15 03:51:34,111] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:51:34,183] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:51:34,199] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:51:34,245] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:51:34,698] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:51:34,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:51:35,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:51:35,437] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:51:35,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:51:35,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:51:36,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:51:36,384] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:51:36,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:51:36,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:51:37,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:51:37,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:51:37,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:51:37,817] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:51:38,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:51:38,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:51:38,327] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:51:38,332] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:51:38,863] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:51:39,299] INFO : Plotted number of elements +rank0 [2026-11-15 03:51:40,521] INFO : Plotted jets +rank0 [2026-11-15 03:51:41,904] INFO : Plotted jet ratio +rank0 [2026-11-15 03:51:54,692] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 03:51:54,692] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:52:55,212] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 03:52:55,213] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:53:30,299] INFO : Step 11000/500000 rank0 | Train Loss: 1.7160 | Train Loss Unweighted: 0.5495 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.4775 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0352 | Regression_eta: 0.0003 | Regression_pt: 0.0354 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.4775 | Regression_cos_phi: 0.1068 | Regression_energy: 0.4385 | Regression_eta: 0.0391 | Regression_pt: 0.4533 | Regression_sin_phi: 0.1466 | LR: 2.63e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4640s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0384 | max_optimizer_step: 0.0606 | max_tensorboard_logging: 0.1258 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2977.6 | uss_MB: 1750.6 | child_rss_MB: 22741.2 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:53:30,301] INFO : Running validation on rank0 +rank0 [2026-11-15 03:53:30,301] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:53:30,301] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:53:30,420] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 03:53:30,524] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:53:30,624] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:53:30,692] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:53:30,759] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:53:30,825] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:53:30,892] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:53:30,959] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:53:31,069] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:53:31,180] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:53:31,290] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:53:31,358] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:53:31,423] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:53:31,487] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:53:31,552] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:53:31,616] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:53:31,713] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:53:31,719] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:53:31,721] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:53:31,723] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:53:31,724] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:53:31,727] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:53:31,728] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:53:31,730] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:53:31,732] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:53:31,732] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:53:31,734] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:53:31,736] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:53:31,737] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:53:31,737] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:53:31,741] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:53:31,743] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:53:33,147] INFO : VALIDATION | Step=11000/500000 | Train Loss=1.7160 | Valid Loss=8.6814 | Train Loss Unweighted=0.5495 | Valid Loss Unweighted=1.2924 | Stale=3 | ETA=961.1m +rank0 [2026-11-15 03:53:33,147] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:53:33,222] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:53:33,225] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:53:33,271] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:53:33,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:53:33,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:53:34,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:53:34,450] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:53:34,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:53:34,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:53:35,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:53:35,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:53:35,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:53:35,884] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:53:36,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:53:36,368] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:53:36,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:53:36,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:53:37,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:53:37,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:53:37,348] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:53:37,354] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:53:37,892] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:53:38,306] INFO : Plotted number of elements +rank0 [2026-11-15 03:53:39,941] INFO : Plotted jets +rank0 [2026-11-15 03:53:40,875] INFO : Plotted jet ratio +rank0 [2026-11-15 03:54:07,061] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 03:54:07,061] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:55:07,640] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 03:55:07,640] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:55:29,358] INFO : Step 12000/500000 rank0 | Train Loss: 1.8046 | Train Loss Unweighted: 1.0903 | Train Loss Components: Classification: 0.0001 | Classification_binary: 1.0508 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0184 | Regression_eta: 0.0002 | Regression_pt: 0.0203 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0071 | Classification_binary: 1.0508 | Regression_cos_phi: 0.0614 | Regression_energy: 0.2295 | Regression_eta: 0.0268 | Regression_pt: 0.2595 | Regression_sin_phi: 0.1695 | LR: 2.75e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4687s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0375 | max_loss: 0.0028 | max_backward: 0.0393 | max_optimizer_step: 0.0619 | max_tensorboard_logging: 0.1292 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2994.6 | uss_MB: 1767.6 | child_rss_MB: 22740.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:55:29,359] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 03:55:29,427] INFO : Running validation on rank0 +rank0 [2026-11-15 03:55:29,427] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:55:29,427] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:55:29,544] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 03:55:29,648] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:55:29,747] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:55:29,815] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:55:29,880] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:55:29,943] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:55:30,008] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:55:30,074] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:55:30,184] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:55:30,286] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:55:30,387] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:55:30,453] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:55:30,520] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:55:30,586] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:55:30,652] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:55:30,719] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:55:30,819] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:55:30,825] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:55:30,827] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:55:30,829] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:55:30,831] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:55:30,833] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:55:30,834] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:55:30,834] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:55:30,838] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:55:30,838] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:55:30,840] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:55:30,842] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:55:30,843] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:55:30,845] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:55:30,847] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:55:30,849] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:55:32,256] INFO : VALIDATION | Step=12000/500000 | Train Loss=1.8046 | Valid Loss=8.4077 | Train Loss Unweighted=1.0903 | Valid Loss Unweighted=1.3178 | Stale=4 | ETA=959.9m +rank0 [2026-11-15 03:55:32,257] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:55:32,330] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:55:32,334] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:55:32,381] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:55:32,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:55:33,074] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:55:33,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:55:33,566] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:55:33,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:55:34,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:55:34,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:55:34,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:55:34,745] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:55:34,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:55:35,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:55:35,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:55:35,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:55:35,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:55:36,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:55:36,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:55:36,446] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:55:36,451] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:55:36,983] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:55:37,417] INFO : Plotted number of elements +rank0 [2026-11-15 03:55:38,652] INFO : Plotted jets +rank0 [2026-11-15 03:55:40,014] INFO : Plotted jet ratio +rank0 [2026-11-15 03:56:19,637] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 03:56:19,637] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:57:20,005] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 03:57:20,005] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:57:28,433] INFO : Step 13000/500000 rank0 | Train Loss: 1.1523 | Train Loss Unweighted: 0.4835 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.4391 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0220 | Regression_eta: 0.0001 | Regression_pt: 0.0218 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0187 | Classification_binary: 0.4391 | Regression_cos_phi: 0.0335 | Regression_energy: 0.2745 | Regression_eta: 0.0214 | Regression_pt: 0.2786 | Regression_sin_phi: 0.0865 | LR: 2.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4965s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0394 | max_loss: 0.0028 | max_backward: 0.0397 | max_optimizer_step: 0.0644 | max_tensorboard_logging: 0.1327 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3029.2 | uss_MB: 1802.2 | child_rss_MB: 22740.5 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:57:28,434] INFO : Running validation on rank0 +rank0 [2026-11-15 03:57:28,434] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:57:28,435] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:57:28,554] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 03:57:28,660] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:57:28,760] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:57:28,830] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:57:28,897] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:57:28,962] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:57:29,029] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:57:29,096] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:57:29,207] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:57:29,317] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:57:29,429] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:57:29,495] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:57:29,561] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:57:29,627] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:57:29,696] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:57:29,763] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:57:29,864] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:57:29,870] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:57:29,872] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:57:29,873] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:57:29,876] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:57:29,878] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:57:29,878] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:57:29,880] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:57:29,882] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:57:29,883] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:57:29,885] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:57:29,887] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:57:29,887] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:57:29,889] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:57:29,891] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:57:29,893] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:57:31,291] INFO : VALIDATION | Step=13000/500000 | Train Loss=1.1523 | Valid Loss=8.7483 | Train Loss Unweighted=0.4835 | Valid Loss Unweighted=1.3270 | Stale=5 | ETA=958.6m +rank0 [2026-11-15 03:57:31,291] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:57:31,365] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:57:31,369] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:57:31,414] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:57:31,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:57:32,098] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:57:32,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:57:32,586] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:57:32,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:57:33,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:57:33,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:57:33,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:57:33,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:57:33,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:57:34,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:57:34,477] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:57:34,717] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:57:34,952] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:57:35,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:57:35,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:57:35,457] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:57:35,462] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:57:35,992] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:57:36,414] INFO : Plotted number of elements +rank0 [2026-11-15 03:57:37,638] INFO : Plotted jets +rank0 [2026-11-15 03:57:38,992] INFO : Plotted jet ratio +rank0 [2026-11-15 03:58:31,732] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 03:58:31,732] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:59:27,184] INFO : Step 14000/500000 rank0 | Train Loss: 2.0174 | Train Loss Unweighted: 0.5081 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.3974 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0560 | Regression_eta: 0.0003 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0184 | Classification_binary: 0.3974 | Regression_cos_phi: 0.0326 | Regression_energy: 0.6985 | Regression_eta: 0.0417 | Regression_pt: 0.6890 | Regression_sin_phi: 0.1397 | LR: 3.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4972s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0398 | max_optimizer_step: 0.0611 | max_tensorboard_logging: 0.1344 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3056.3 | uss_MB: 1829.3 | child_rss_MB: 22738.7 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:59:27,185] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 03:59:27,250] INFO : Running validation on rank0 +rank0 [2026-11-15 03:59:27,251] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:59:27,251] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:59:27,370] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 03:59:27,474] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:59:27,577] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:59:27,645] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:59:27,710] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:59:27,774] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:59:27,839] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:59:27,905] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:59:28,020] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:59:28,130] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:59:28,240] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:59:28,309] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:59:28,376] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:59:28,441] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:59:28,507] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:59:28,573] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:59:28,677] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:59:28,683] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:59:28,685] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:59:28,687] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:59:28,689] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:59:28,691] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:59:28,692] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:59:28,694] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:59:28,696] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:59:28,697] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:59:28,698] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:59:28,699] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:59:28,701] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:59:28,703] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:59:28,705] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:59:28,707] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:59:30,112] INFO : VALIDATION | Step=14000/500000 | Train Loss=2.0174 | Valid Loss=9.0274 | Train Loss Unweighted=0.5081 | Valid Loss Unweighted=1.3847 | Stale=6 | ETA=957.0m +rank0 [2026-11-15 03:59:30,112] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:59:30,186] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:59:30,191] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:59:30,238] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:59:30,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:59:30,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:59:31,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:59:31,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:59:31,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:59:31,923] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:59:32,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:59:32,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:59:32,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:59:32,874] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:59:33,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:59:33,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:59:33,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:59:33,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:59:34,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:59:34,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:59:34,342] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:59:34,348] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:59:34,877] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:59:35,294] INFO : Plotted number of elements +rank0 [2026-11-15 03:59:36,508] INFO : Plotted jets +rank0 [2026-11-15 03:59:37,866] INFO : Plotted jet ratio +rank0 [2026-11-15 03:59:43,529] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 03:59:43,529] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:00:44,202] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 04:00:44,202] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:01:26,000] INFO : Step 15000/500000 rank0 | Train Loss: 2.1749 | Train Loss Unweighted: 0.7710 | Train Loss Components: Classification: 0.0025 | Classification_binary: 0.7017 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0335 | Regression_eta: 0.0004 | Regression_pt: 0.0319 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.2343 | Classification_binary: 0.7017 | Regression_cos_phi: 0.1773 | Regression_energy: 0.4174 | Regression_eta: 0.0642 | Regression_pt: 0.4090 | Regression_sin_phi: 0.1711 | LR: 3.17e-05 | DataLoad Time: 0.0004s | Model Forward Time: 0.4669s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0344 | max_loss: 0.0025 | max_backward: 0.0380 | max_optimizer_step: 0.0605 | max_tensorboard_logging: 0.1270 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3079.3 | uss_MB: 1852.3 | child_rss_MB: 22737.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:01:26,001] INFO : Running validation on rank0 +rank0 [2026-11-15 04:01:26,002] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:01:26,002] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:01:26,125] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 04:01:26,233] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:01:26,335] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:01:26,402] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:01:26,468] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:01:26,534] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:01:26,601] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:01:26,668] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:01:26,780] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:01:26,886] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:01:26,993] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:01:27,060] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:01:27,124] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:01:27,187] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:01:27,250] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:01:27,314] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:01:27,413] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:01:27,419] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:01:27,421] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:01:27,423] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:01:27,425] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:01:27,427] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:01:27,428] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:01:27,428] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:01:27,430] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:01:27,432] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:01:27,434] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:01:27,436] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:01:27,437] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:01:27,437] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:01:27,440] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:01:27,443] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:01:28,826] INFO : VALIDATION | Step=15000/500000 | Train Loss=2.1749 | Valid Loss=8.9581 | Train Loss Unweighted=0.7710 | Valid Loss Unweighted=1.4120 | Stale=7 | ETA=955.4m +rank0 [2026-11-15 04:01:28,826] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:01:28,898] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:01:28,909] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:01:28,956] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:01:29,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:01:29,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:01:29,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:01:30,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:01:30,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:01:30,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:01:30,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:01:31,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:01:31,335] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:01:31,574] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:01:31,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:01:32,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:01:32,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:01:32,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:01:32,751] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:01:32,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:01:33,049] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:01:33,058] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:01:33,598] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:01:34,005] INFO : Plotted number of elements +rank0 [2026-11-15 04:01:35,259] INFO : Plotted jets +rank0 [2026-11-15 04:01:36,624] INFO : Plotted jet ratio +rank0 [2026-11-15 04:01:55,726] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 04:01:55,726] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:02:56,506] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 04:02:56,506] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:03:25,590] INFO : Step 16000/500000 rank0 | Train Loss: 0.8304 | Train Loss Unweighted: 0.4566 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.4323 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0118 | Regression_eta: 0.0001 | Regression_pt: 0.0118 | Regression_sin_phi: 0.0000 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0301 | Classification_binary: 0.4323 | Regression_cos_phi: 0.0248 | Regression_energy: 0.1476 | Regression_eta: 0.0186 | Regression_pt: 0.1508 | Regression_sin_phi: 0.0261 | LR: 3.33e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4829s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0378 | max_optimizer_step: 0.0605 | max_tensorboard_logging: 0.1344 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3102.5 | uss_MB: 1875.6 | child_rss_MB: 22737.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:03:25,592] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 04:03:25,660] INFO : Running validation on rank0 +rank0 [2026-11-15 04:03:25,660] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:03:25,660] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:03:25,789] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 04:03:25,895] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:03:25,998] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:03:26,066] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:03:26,133] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:03:26,199] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:03:26,266] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:03:26,332] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:03:26,441] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:03:26,549] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:03:26,657] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:03:26,724] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:03:26,790] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:03:26,853] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:03:26,917] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:03:26,982] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:03:27,081] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:03:27,087] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:03:27,089] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:03:27,091] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:03:27,093] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:03:27,094] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:03:27,095] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:03:27,097] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:03:27,099] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:03:27,100] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:03:27,102] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:03:27,104] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:03:27,104] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:03:27,106] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:03:27,108] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:03:27,110] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:03:28,517] INFO : VALIDATION | Step=16000/500000 | Train Loss=0.8304 | Valid Loss=9.1480 | Train Loss Unweighted=0.4566 | Valid Loss Unweighted=1.3879 | Stale=8 | ETA=954.2m +rank0 [2026-11-15 04:03:28,517] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:03:28,601] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:03:28,604] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:03:28,646] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:03:29,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:03:29,330] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:03:29,566] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:03:29,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:03:30,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:03:30,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:03:30,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:03:30,752] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:03:30,991] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:03:31,229] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:03:31,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:03:31,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:03:31,928] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:03:32,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:03:32,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:03:32,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:03:32,678] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 04:03:32,683] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:03:33,211] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:03:33,617] INFO : Plotted number of elements +rank0 [2026-11-15 04:03:34,865] INFO : Plotted jets +rank0 [2026-11-15 04:03:36,249] INFO : Plotted jet ratio +rank0 [2026-11-15 04:04:08,756] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 04:04:08,756] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:05:09,552] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 04:05:09,553] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:05:25,138] INFO : Step 17000/500000 rank0 | Train Loss: 0.7870 | Train Loss Unweighted: 0.3152 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.2858 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0145 | Regression_eta: 0.0002 | Regression_pt: 0.0143 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0221 | Classification_binary: 0.2858 | Regression_cos_phi: 0.0315 | Regression_energy: 0.1803 | Regression_eta: 0.0285 | Regression_pt: 0.1836 | Regression_sin_phi: 0.0552 | LR: 3.51e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4971s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0358 | max_loss: 0.0027 | max_backward: 0.0389 | max_optimizer_step: 0.0652 | max_tensorboard_logging: 0.1343 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3126.6 | uss_MB: 1899.6 | child_rss_MB: 22762.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:05:25,140] INFO : Running validation on rank0 +rank0 [2026-11-15 04:05:25,140] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:05:25,140] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:05:25,261] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 04:05:25,369] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:05:25,470] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:05:25,537] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:05:25,602] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:05:25,667] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:05:25,734] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:05:25,800] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:05:25,908] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:05:26,017] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:05:26,126] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:05:26,198] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:05:26,267] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:05:26,334] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:05:26,401] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:05:26,470] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:05:26,575] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:05:26,581] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:05:26,583] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:05:26,585] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:05:26,587] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:05:26,589] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:05:26,590] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:05:26,590] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:05:26,593] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:05:26,594] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:05:26,596] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:05:26,598] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:05:26,599] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:05:26,601] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:05:26,603] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:05:26,605] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:05:28,032] INFO : VALIDATION | Step=17000/500000 | Train Loss=0.7870 | Valid Loss=9.5913 | Train Loss Unweighted=0.3152 | Valid Loss Unweighted=1.4813 | Stale=9 | ETA=952.8m +rank0 [2026-11-15 04:05:28,033] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:05:28,107] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:05:28,110] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:05:28,158] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:05:28,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:05:28,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:05:29,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:05:29,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:05:29,582] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:05:29,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:05:30,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:05:30,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:05:30,533] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:05:30,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:05:31,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:05:31,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:05:31,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:05:31,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:05:31,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:05:32,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:05:32,248] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:05:32,256] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:05:32,783] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:05:33,202] INFO : Plotted number of elements +rank0 [2026-11-15 04:05:34,371] INFO : Plotted jets +rank0 [2026-11-15 04:05:35,734] INFO : Plotted jet ratio +rank0 [2026-11-15 04:06:21,838] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 04:06:21,838] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:07:22,887] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 04:07:22,887] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:07:24,988] INFO : Step 18000/500000 rank0 | Train Loss: 0.8505 | Train Loss Unweighted: 0.1724 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1249 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0234 | Regression_eta: 0.0002 | Regression_pt: 0.0237 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0023 | Classification_binary: 0.1249 | Regression_cos_phi: 0.0582 | Regression_energy: 0.2914 | Regression_eta: 0.0271 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0437 | LR: 3.69e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4833s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0026 | max_backward: 0.0388 | max_optimizer_step: 0.0626 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3155.6 | uss_MB: 1928.7 | child_rss_MB: 22767.3 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 04:07:24,990] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 04:07:25,057] INFO : Running validation on rank0 +rank0 [2026-11-15 04:07:25,057] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:07:25,057] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:07:25,176] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 04:07:25,282] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:07:25,383] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:07:25,451] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:07:25,517] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:07:25,582] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:07:25,647] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:07:25,712] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:07:25,821] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:07:25,930] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:07:26,040] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:07:26,108] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:07:26,175] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:07:26,241] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:07:26,309] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:07:26,376] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:07:26,478] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:07:26,484] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:07:26,486] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:07:26,489] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:07:26,490] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:07:26,492] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:07:26,493] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:07:26,495] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:07:26,497] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:07:26,498] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:07:26,500] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:07:26,501] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:07:26,502] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:07:26,504] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:07:26,506] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:07:26,508] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:07:27,927] INFO : VALIDATION | Step=18000/500000 | Train Loss=0.8505 | Valid Loss=9.2889 | Train Loss Unweighted=0.1724 | Valid Loss Unweighted=1.4915 | Stale=10 | ETA=951.5m +rank0 [2026-11-15 04:07:27,928] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:07:28,002] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:07:28,007] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:07:28,055] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:07:28,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:07:28,764] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:07:29,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:07:29,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:07:29,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:07:29,745] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:07:29,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:07:30,227] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:07:30,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:07:30,698] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:07:30,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:07:31,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:07:31,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:07:31,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:07:31,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:07:32,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:07:32,180] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:07:32,193] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:07:32,721] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:07:33,144] INFO : Plotted number of elements +rank0 [2026-11-15 04:07:34,381] INFO : Plotted jets +rank0 [2026-11-15 04:07:35,758] INFO : Plotted jet ratio +rank0 [2026-11-15 04:08:34,942] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 04:08:34,942] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:09:24,036] INFO : Step 19000/500000 rank0 | Train Loss: 1.3460 | Train Loss Unweighted: 0.5704 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.5411 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0137 | Regression_eta: 0.0003 | Regression_pt: 0.0140 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.5411 | Regression_cos_phi: 0.1455 | Regression_energy: 0.1706 | Regression_eta: 0.0411 | Regression_pt: 0.1792 | Regression_sin_phi: 0.2226 | LR: 3.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4742s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0345 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0603 | max_tensorboard_logging: 0.1305 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3186.5 | uss_MB: 1959.5 | child_rss_MB: 22738.7 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 04:09:24,038] INFO : Running validation on rank0 +rank0 [2026-11-15 04:09:24,038] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:09:24,038] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:09:24,167] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 04:09:24,270] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:09:24,372] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:09:24,441] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:09:24,508] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:09:24,575] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:09:24,642] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:09:24,708] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:09:24,816] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:09:24,925] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:09:25,033] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:09:25,099] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:09:25,163] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:09:25,226] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:09:25,291] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:09:25,356] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:09:25,454] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:09:25,460] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:09:25,462] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:09:25,465] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:09:25,465] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:09:25,468] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:09:25,469] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:09:25,471] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:09:25,473] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:09:25,474] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:09:25,475] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:09:25,476] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:09:25,478] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:09:25,480] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:09:25,482] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:09:25,484] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:09:26,872] INFO : VALIDATION | Step=19000/500000 | Train Loss=1.3460 | Valid Loss=10.1494 | Train Loss Unweighted=0.5704 | Valid Loss Unweighted=1.5967 | Stale=11 | ETA=949.7m +rank0 [2026-11-15 04:09:26,872] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:09:26,947] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:09:26,962] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:09:27,010] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:09:27,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:09:27,725] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:09:27,963] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:09:28,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:09:28,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:09:28,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:09:28,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:09:29,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:09:29,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:09:29,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:09:29,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:09:30,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:09:30,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:09:30,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:09:30,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:09:31,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:09:31,085] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 04:09:31,094] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:09:31,631] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:09:32,041] INFO : Plotted number of elements +rank0 [2026-11-15 04:09:33,207] INFO : Plotted jets +rank0 [2026-11-15 04:09:34,555] INFO : Plotted jet ratio +rank0 [2026-11-15 04:09:46,491] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 04:09:46,491] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:10:46,913] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 04:10:46,913] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:11:23,144] INFO : Step 20000/500000 rank0 | Train Loss: 0.7822 | Train Loss Unweighted: 0.2006 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1690 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0158 | Regression_eta: 0.0005 | Regression_pt: 0.0148 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0023 | Classification_binary: 0.1690 | Regression_cos_phi: 0.0572 | Regression_energy: 0.1968 | Regression_eta: 0.0792 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0877 | LR: 4.07e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4959s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0027 | max_backward: 0.0396 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.1355 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3133.3 | uss_MB: 1906.4 | child_rss_MB: 22740.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:11:23,146] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 04:11:23,214] INFO : Running validation on rank0 +rank0 [2026-11-15 04:11:23,214] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:11:23,214] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:11:23,331] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 04:11:23,437] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:11:23,537] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:11:23,604] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:11:23,668] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:11:23,732] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:11:23,799] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:11:23,865] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:11:23,974] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:11:24,086] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:11:24,196] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:11:24,264] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:11:24,330] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:11:24,394] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:11:24,458] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:11:24,522] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:11:24,624] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:11:24,630] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:11:24,632] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:11:24,634] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:11:24,636] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:11:24,638] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:11:24,639] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:11:24,640] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:11:24,643] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:11:24,643] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:11:24,645] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:11:24,647] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:11:24,648] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:11:24,650] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:11:24,650] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:11:24,654] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:11:26,041] INFO : VALIDATION | Step=20000/500000 | Train Loss=0.7822 | Valid Loss=9.7266 | Train Loss Unweighted=0.2006 | Valid Loss Unweighted=1.5992 | Stale=12 | ETA=948.0m +rank0 [2026-11-15 04:11:26,042] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:11:26,116] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:11:26,125] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:11:26,172] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:11:26,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:11:26,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:11:27,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:11:27,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:11:27,581] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:11:27,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:11:28,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:11:28,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:11:28,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:11:28,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:11:29,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:11:29,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:11:29,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:11:29,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:11:29,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:11:30,176] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:11:30,237] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 04:11:30,242] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:11:30,774] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:11:31,195] INFO : Plotted number of elements +rank0 [2026-11-15 04:11:32,348] INFO : Plotted jets +rank0 [2026-11-15 04:11:33,722] INFO : Plotted jet ratio +rank0 [2026-11-15 04:11:58,936] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 04:11:58,937] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:12:59,591] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 04:12:59,591] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:13:22,136] INFO : Step 21000/500000 rank0 | Train Loss: 2.7784 | Train Loss Unweighted: 0.3121 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.1191 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0974 | Regression_eta: 0.0002 | Regression_pt: 0.0947 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0288 | Classification_binary: 0.1191 | Regression_cos_phi: 0.0292 | Regression_energy: 1.2142 | Regression_eta: 0.0252 | Regression_pt: 1.2127 | Regression_sin_phi: 0.1493 | LR: 4.28e-05 | DataLoad Time: 0.0004s | Model Forward Time: 0.4905s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0026 | max_backward: 0.0439 | max_optimizer_step: 0.0652 | max_tensorboard_logging: 0.1293 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3233.2 | uss_MB: 2006.2 | child_rss_MB: 22739.8 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:13:22,137] INFO : Running validation on rank0 +rank0 [2026-11-15 04:13:22,137] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:13:22,137] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:13:22,258] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 04:13:22,363] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:13:22,465] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:13:22,532] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:13:22,597] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:13:22,662] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:13:22,728] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:13:22,794] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:13:22,907] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:13:23,017] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:13:23,127] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:13:23,196] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:13:23,262] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:13:23,326] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:13:23,393] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:13:23,460] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:13:23,571] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:13:23,578] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:13:23,579] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:13:23,583] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:13:23,585] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:13:23,585] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:13:23,587] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:13:23,589] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:13:23,591] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:13:23,592] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:13:23,594] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:13:23,595] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:13:23,597] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:13:23,599] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:13:23,601] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:13:23,603] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:13:25,029] INFO : VALIDATION | Step=21000/500000 | Train Loss=2.7784 | Valid Loss=9.9692 | Train Loss Unweighted=0.3121 | Valid Loss Unweighted=1.6642 | Stale=13 | ETA=946.2m +rank0 [2026-11-15 04:13:25,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:13:25,102] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:13:25,105] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:13:25,153] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:13:25,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:13:25,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:13:26,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:13:26,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:13:26,600] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:13:26,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:13:27,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:13:27,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:13:27,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:13:27,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:13:28,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:13:28,267] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:13:28,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:13:28,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:13:28,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:13:29,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:13:29,272] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:13:29,277] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:13:29,806] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:13:30,226] INFO : Plotted number of elements +rank0 [2026-11-15 04:13:31,391] INFO : Plotted jets +rank0 [2026-11-15 04:13:32,769] INFO : Plotted jet ratio +rank0 [2026-11-15 04:14:11,671] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 04:14:11,671] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:15:12,629] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 04:15:12,630] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:15:21,907] INFO : Step 22000/500000 rank0 | Train Loss: 1.2712 | Train Loss Unweighted: 0.4533 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.4048 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0245 | Regression_eta: 0.0003 | Regression_pt: 0.0231 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0061 | Classification_binary: 0.4048 | Regression_cos_phi: 0.1282 | Regression_energy: 0.3048 | Regression_eta: 0.0456 | Regression_pt: 0.2953 | Regression_sin_phi: 0.0864 | LR: 4.50e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4917s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0026 | max_backward: 0.0384 | max_optimizer_step: 0.0649 | max_tensorboard_logging: 0.1322 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3259.7 | uss_MB: 2032.7 | child_rss_MB: 22739.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:15:21,909] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 04:15:21,978] INFO : Running validation on rank0 +rank0 [2026-11-15 04:15:21,978] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:15:21,978] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:15:22,099] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 04:15:22,203] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:15:22,303] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:15:22,371] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:15:22,437] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:15:22,501] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:15:22,566] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:15:22,631] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:15:22,743] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:15:22,854] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:15:22,963] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:15:23,033] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:15:23,099] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:15:23,165] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:15:23,231] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:15:23,298] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:15:23,400] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:15:23,406] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:15:23,408] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:15:23,411] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:15:23,411] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:15:23,414] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:15:23,415] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:15:23,417] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:15:23,419] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:15:23,420] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:15:23,422] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:15:23,424] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:15:23,424] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:15:23,426] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:15:23,428] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:15:23,431] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:15:24,855] INFO : VALIDATION | Step=22000/500000 | Train Loss=1.2712 | Valid Loss=10.0428 | Train Loss Unweighted=0.4533 | Valid Loss Unweighted=1.7843 | Stale=14 | ETA=944.7m +rank0 [2026-11-15 04:15:24,856] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:15:24,931] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:15:24,938] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:15:24,986] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:15:25,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:15:25,700] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:15:25,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:15:26,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:15:26,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:15:26,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:15:26,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:15:27,153] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:15:27,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:15:27,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:15:27,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:15:28,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:15:28,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:15:28,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:15:28,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:15:29,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:15:29,095] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:15:29,101] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:15:29,627] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:15:30,048] INFO : Plotted number of elements +rank0 [2026-11-15 04:15:31,206] INFO : Plotted jets +rank0 [2026-11-15 04:15:32,553] INFO : Plotted jet ratio +rank0 [2026-11-15 04:16:24,530] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 04:16:24,530] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:17:20,872] INFO : Step 23000/500000 rank0 | Train Loss: 11.8012 | Train Loss Unweighted: 0.8686 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.7498 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0424 | Regression_eta: 0.0207 | Regression_pt: 0.0417 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.7498 | Regression_cos_phi: 0.9199 | Regression_energy: 0.5288 | Regression_eta: 3.0652 | Regression_pt: 0.5340 | Regression_sin_phi: 5.9522 | LR: 4.73e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4750s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0382 | max_optimizer_step: 0.0614 | max_tensorboard_logging: 0.1275 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3213.1 | uss_MB: 1986.2 | child_rss_MB: 22738.9 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 04:17:20,874] INFO : Running validation on rank0 +rank0 [2026-11-15 04:17:20,874] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:17:20,874] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:17:20,994] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 04:17:21,100] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:17:21,200] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:17:21,269] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:17:21,336] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:17:21,401] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:17:21,468] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:17:21,535] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:17:21,646] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:17:21,750] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:17:21,852] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:17:21,920] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:17:21,983] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:17:22,047] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:17:22,113] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:17:22,177] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:17:22,282] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:17:22,288] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:17:22,290] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:17:22,293] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:17:22,295] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:17:22,296] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:17:22,297] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:17:22,299] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:17:22,301] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:17:22,302] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:17:22,304] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:17:22,304] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:17:22,306] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:17:22,308] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:17:22,310] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:17:22,312] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:17:23,716] INFO : VALIDATION | Step=23000/500000 | Train Loss=11.8012 | Valid Loss=10.5621 | Train Loss Unweighted=0.8686 | Valid Loss Unweighted=1.9085 | Stale=15 | ETA=942.8m +rank0 [2026-11-15 04:17:23,716] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:17:23,790] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:17:23,804] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:17:23,852] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:17:24,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:17:24,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:17:24,805] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:17:25,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:17:25,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:17:25,524] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:17:25,757] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:17:25,993] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:17:26,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:17:26,473] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:17:26,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:17:26,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:17:27,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:17:27,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:17:27,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:17:27,874] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:17:27,930] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 04:17:27,938] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:17:28,477] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:17:28,893] INFO : Plotted number of elements +rank0 [2026-11-15 04:17:30,067] INFO : Plotted jets +rank0 [2026-11-15 04:17:31,434] INFO : Plotted jet ratio +rank0 [2026-11-15 04:17:36,278] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 04:17:36,278] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:18:36,459] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 04:18:36,460] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:19:19,435] INFO : Step 24000/500000 rank0 | Train Loss: 1.5245 | Train Loss Unweighted: 0.3563 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.3078 | Regression_cos_phi: 0.0011 | Regression_energy: 0.0250 | Regression_eta: 0.0008 | Regression_pt: 0.0212 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0013 | Classification_binary: 0.3078 | Regression_cos_phi: 0.3137 | Regression_energy: 0.3121 | Regression_eta: 0.1165 | Regression_pt: 0.2716 | Regression_sin_phi: 0.2015 | LR: 4.97e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5212s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0377 | max_loss: 0.0028 | max_backward: 0.0394 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1376 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3308.9 | uss_MB: 2082.0 | child_rss_MB: 22737.5 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 04:19:19,437] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 04:19:19,506] INFO : Running validation on rank0 +rank0 [2026-11-15 04:19:19,506] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:19:19,506] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:19:19,626] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 04:19:19,729] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:19:19,828] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:19:19,897] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:19:19,964] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:19:20,030] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:19:20,098] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:19:20,164] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:19:20,278] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:19:20,387] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:19:20,493] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:19:20,560] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:19:20,626] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:19:20,690] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:19:20,756] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:19:20,824] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:19:20,930] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:19:20,936] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:19:20,939] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:19:20,939] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:19:20,941] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:19:20,945] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:19:20,945] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:19:20,947] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:19:20,949] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:19:20,950] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:19:20,952] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:19:20,953] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:19:20,954] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:19:20,955] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:19:20,957] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:19:20,960] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:19:22,352] INFO : VALIDATION | Step=24000/500000 | Train Loss=1.5245 | Valid Loss=10.7359 | Train Loss Unweighted=0.3563 | Valid Loss Unweighted=2.1115 | Stale=16 | ETA=940.9m +rank0 [2026-11-15 04:19:22,353] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:19:22,426] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:19:22,433] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:19:22,481] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:19:22,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:19:23,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:19:23,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:19:23,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:19:23,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:19:24,156] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:19:24,394] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:19:24,630] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:19:24,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:19:25,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:19:25,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:19:25,578] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:19:25,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:19:26,057] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:19:26,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:19:26,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:19:26,572] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 04:19:26,577] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:19:27,101] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:19:27,517] INFO : Plotted number of elements +rank0 [2026-11-15 04:19:28,671] INFO : Plotted jets +rank0 [2026-11-15 04:19:30,030] INFO : Plotted jet ratio +rank0 [2026-11-15 04:19:48,169] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 04:19:48,169] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:20:48,547] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 04:20:48,547] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:21:18,063] INFO : Step 25000/500000 rank0 | Train Loss: 1.9117 | Train Loss Unweighted: 0.7077 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.6707 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0201 | Regression_eta: 0.0037 | Regression_pt: 0.0126 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0003 | Classification_binary: 0.6707 | Regression_cos_phi: 0.0518 | Regression_energy: 0.2508 | Regression_eta: 0.5486 | Regression_pt: 0.1613 | Regression_sin_phi: 0.2283 | LR: 5.22e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4771s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0358 | max_loss: 0.0026 | max_backward: 0.0389 | max_optimizer_step: 0.0626 | max_tensorboard_logging: 0.1256 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3328.1 | uss_MB: 2101.2 | child_rss_MB: 22730.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 04:21:18,064] INFO : Running validation on rank0 +rank0 [2026-11-15 04:21:18,065] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:21:18,065] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:21:18,186] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 04:21:18,290] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:21:18,391] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:21:18,460] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:21:18,524] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:21:18,588] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:21:18,657] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:21:18,725] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:21:18,839] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:21:18,952] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:21:19,062] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:21:19,132] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:21:19,199] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:21:19,265] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:21:19,331] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:21:19,398] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:21:19,502] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:21:19,508] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:21:19,509] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:21:19,512] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:21:19,514] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:21:19,516] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:21:19,517] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:21:19,517] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:21:19,519] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:21:19,521] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:21:19,523] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:21:19,527] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:21:19,527] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:21:19,529] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:21:19,531] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:21:19,534] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:21:20,955] INFO : VALIDATION | Step=25000/500000 | Train Loss=1.9117 | Valid Loss=10.4990 | Train Loss Unweighted=0.7077 | Valid Loss Unweighted=1.9100 | Stale=17 | ETA=938.9m +rank0 [2026-11-15 04:21:20,955] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:21:21,030] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:21:21,034] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:21:21,082] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:21:21,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:21:21,789] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:21:22,053] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:21:22,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:21:22,539] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:21:22,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:21:23,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:21:23,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:21:23,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:21:23,733] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:21:23,979] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:21:24,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:21:24,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:21:24,700] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:21:24,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:21:25,149] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:21:25,212] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:21:25,220] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:21:25,752] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:21:26,177] INFO : Plotted number of elements +rank0 [2026-11-15 04:21:27,341] INFO : Plotted jets +rank0 [2026-11-15 04:21:28,725] INFO : Plotted jet ratio +rank0 [2026-11-15 04:22:00,493] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 04:22:00,494] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:23:01,359] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 04:23:01,359] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:23:17,763] INFO : Step 26000/500000 rank0 | Train Loss: 1.1273 | Train Loss Unweighted: 0.5497 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.5133 | Regression_cos_phi: 0.0000 | Regression_energy: 0.0182 | Regression_eta: 0.0008 | Regression_pt: 0.0173 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0007 | Classification_binary: 0.5133 | Regression_cos_phi: 0.0120 | Regression_energy: 0.2264 | Regression_eta: 0.1188 | Regression_pt: 0.2214 | Regression_sin_phi: 0.0348 | LR: 5.47e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.4805s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0028 | max_backward: 0.0385 | max_optimizer_step: 0.0636 | max_tensorboard_logging: 0.1273 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3346.3 | uss_MB: 2119.3 | child_rss_MB: 22740.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 04:23:17,765] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 04:23:17,832] INFO : Running validation on rank0 +rank0 [2026-11-15 04:23:17,832] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:23:17,832] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:23:17,954] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 04:23:18,059] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:23:18,160] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:23:18,228] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:23:18,294] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:23:18,359] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:23:18,425] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:23:18,493] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:23:18,606] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:23:18,710] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:23:18,812] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:23:18,880] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:23:18,946] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:23:19,012] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:23:19,080] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:23:19,145] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:23:19,252] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:23:19,259] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:23:19,261] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:23:19,263] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:23:19,266] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:23:19,268] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:23:19,269] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:23:19,271] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:23:19,273] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:23:19,274] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:23:19,274] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:23:19,277] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:23:19,278] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:23:19,279] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:23:19,282] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:23:19,284] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:23:20,699] INFO : VALIDATION | Step=26000/500000 | Train Loss=1.1273 | Valid Loss=11.0044 | Train Loss Unweighted=0.5497 | Valid Loss Unweighted=2.1101 | Stale=18 | ETA=937.3m +rank0 [2026-11-15 04:23:20,699] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:23:20,773] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:23:20,780] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:23:20,829] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:23:21,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:23:21,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:23:21,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:23:22,067] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:23:22,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:23:22,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:23:22,783] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:23:23,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:23:23,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:23:23,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:23:23,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:23:23,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:23:24,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:23:24,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:23:24,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:23:24,903] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:23:24,964] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:23:24,970] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:23:25,510] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:23:25,930] INFO : Plotted number of elements +rank0 [2026-11-15 04:23:27,083] INFO : Plotted jets +rank0 [2026-11-15 04:23:28,439] INFO : Plotted jet ratio +rank0 [2026-11-15 04:24:13,136] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 04:24:13,136] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:25:13,603] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 04:25:13,603] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:25:16,610] INFO : Step 27000/500000 rank0 | Train Loss: 0.6375 | Train Loss Unweighted: 0.0663 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.0387 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0130 | Regression_eta: 0.0002 | Regression_pt: 0.0135 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0351 | Classification_binary: 0.0387 | Regression_cos_phi: 0.0597 | Regression_energy: 0.1625 | Regression_eta: 0.0340 | Regression_pt: 0.1728 | Regression_sin_phi: 0.1346 | LR: 5.74e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4892s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0388 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.1288 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3381.2 | uss_MB: 2154.3 | child_rss_MB: 22739.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:25:16,611] INFO : Running validation on rank0 +rank0 [2026-11-15 04:25:16,611] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:25:16,611] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:25:16,732] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 04:25:16,835] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:25:16,935] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:25:17,004] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:25:17,070] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:25:17,135] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:25:17,202] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:25:17,270] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:25:17,386] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:25:17,498] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:25:17,607] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:25:17,674] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:25:17,739] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:25:17,803] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:25:17,867] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:25:17,931] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:25:18,038] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:25:18,044] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:25:18,046] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:25:18,048] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:25:18,050] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:25:18,052] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:25:18,053] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:25:18,054] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:25:18,055] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:25:18,057] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:25:18,059] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:25:18,061] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:25:18,062] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:25:18,064] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:25:18,066] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:25:18,068] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:25:19,478] INFO : VALIDATION | Step=27000/500000 | Train Loss=0.6375 | Valid Loss=11.2441 | Train Loss Unweighted=0.0663 | Valid Loss Unweighted=2.2967 | Stale=19 | ETA=935.3m +rank0 [2026-11-15 04:25:19,478] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:25:19,553] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:25:19,556] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:25:19,605] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:25:20,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:25:20,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:25:20,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:25:20,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:25:21,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:25:21,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:25:21,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:25:21,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:25:22,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:25:22,245] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:25:22,482] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:25:22,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:25:22,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:25:23,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:25:23,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:25:23,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:25:23,699] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 04:25:23,707] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:25:24,244] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:25:24,650] INFO : Plotted number of elements +rank0 [2026-11-15 04:25:25,819] INFO : Plotted jets +rank0 [2026-11-15 04:25:27,187] INFO : Plotted jet ratio +rank0 [2026-11-15 04:26:25,519] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 04:26:25,519] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:27:15,359] INFO : Step 28000/500000 rank0 | Train Loss: 0.4505 | Train Loss Unweighted: 0.0397 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0136 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0127 | Regression_eta: 0.0002 | Regression_pt: 0.0130 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0000 | Classification_binary: 0.0136 | Regression_cos_phi: 0.0270 | Regression_energy: 0.1582 | Regression_eta: 0.0274 | Regression_pt: 0.1668 | Regression_sin_phi: 0.0575 | LR: 6.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4831s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0029 | max_backward: 0.0392 | max_optimizer_step: 0.0640 | max_tensorboard_logging: 0.1282 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3401.2 | uss_MB: 2174.3 | child_rss_MB: 22736.9 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 04:27:15,361] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 04:27:15,428] INFO : Running validation on rank0 +rank0 [2026-11-15 04:27:15,428] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:27:15,428] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:27:15,546] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 04:27:15,653] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:27:15,753] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:27:15,822] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:27:15,886] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:27:15,950] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:27:16,016] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:27:16,083] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:27:16,195] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:27:16,308] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:27:16,422] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:27:16,490] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:27:16,557] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:27:16,622] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:27:16,688] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:27:16,755] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:27:16,862] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:27:16,868] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:27:16,870] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:27:16,872] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:27:16,874] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:27:16,876] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:27:16,877] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:27:16,879] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:27:16,881] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:27:16,882] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:27:16,883] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:27:16,885] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:27:16,886] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:27:16,888] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:27:16,890] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:27:16,892] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:27:18,305] INFO : VALIDATION | Step=28000/500000 | Train Loss=0.4505 | Valid Loss=11.3729 | Train Loss Unweighted=0.0397 | Valid Loss Unweighted=2.4492 | Stale=20 | ETA=933.4m +rank0 [2026-11-15 04:27:18,305] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:27:18,381] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:27:18,389] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:27:18,437] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:27:18,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:27:19,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:27:19,393] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:27:19,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:27:19,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:27:20,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:27:20,352] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:27:20,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:27:20,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:27:21,064] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:27:21,307] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:27:21,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:27:21,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:27:22,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:27:22,260] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:27:22,474] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:27:22,541] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:27:22,547] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:27:23,077] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:27:23,499] INFO : Plotted number of elements +rank0 [2026-11-15 04:27:24,652] INFO : Plotted jets +rank0 [2026-11-15 04:27:26,050] INFO : Plotted jet ratio +rank0 [2026-11-15 04:27:37,171] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 04:27:37,171] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:28:37,537] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 04:28:37,537] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:29:14,294] INFO : Step 29000/500000 rank0 | Train Loss: 0.2890 | Train Loss Unweighted: 0.0344 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0188 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0079 | Regression_eta: 0.0002 | Regression_pt: 0.0074 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0004 | Classification_binary: 0.0188 | Regression_cos_phi: 0.0216 | Regression_energy: 0.0990 | Regression_eta: 0.0232 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0319 | LR: 6.29e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4800s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0388 | max_optimizer_step: 0.0617 | max_tensorboard_logging: 0.1268 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3430.9 | uss_MB: 2204.0 | child_rss_MB: 22740.8 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:29:14,296] INFO : Running validation on rank0 +rank0 [2026-11-15 04:29:14,296] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:29:14,296] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:29:14,416] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 04:29:14,519] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:29:14,620] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:29:14,688] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:29:14,755] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:29:14,821] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:29:14,889] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:29:14,956] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:29:15,069] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:29:15,176] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:29:15,283] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:29:15,350] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:29:15,413] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:29:15,476] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:29:15,540] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:29:15,604] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:29:15,709] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:29:15,716] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:29:15,718] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:29:15,719] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:29:15,722] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:29:15,723] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:29:15,724] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:29:15,725] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:29:15,728] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:29:15,729] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:29:15,729] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:29:15,733] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:29:15,733] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:29:15,735] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:29:15,737] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:29:15,739] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:29:17,136] INFO : VALIDATION | Step=29000/500000 | Train Loss=0.2890 | Valid Loss=12.1393 | Train Loss Unweighted=0.0344 | Valid Loss Unweighted=2.6877 | Stale=21 | ETA=931.5m +rank0 [2026-11-15 04:29:17,136] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:29:17,210] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:29:17,214] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:29:17,276] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:29:17,765] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:29:18,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:29:18,246] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:29:18,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:29:18,721] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:29:18,955] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:29:19,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:29:19,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:29:19,681] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:29:19,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:29:20,173] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:29:20,408] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:29:20,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:29:20,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:29:21,113] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:29:21,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:29:21,389] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:29:21,398] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:29:21,929] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:29:22,336] INFO : Plotted number of elements +rank0 [2026-11-15 04:29:23,501] INFO : Plotted jets +rank0 [2026-11-15 04:29:24,871] INFO : Plotted jet ratio +rank0 [2026-11-15 04:29:49,488] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 04:29:49,488] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:30:50,407] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 04:30:50,407] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:31:13,956] INFO : Step 30000/500000 rank0 | Train Loss: 0.4761 | Train Loss Unweighted: 0.0908 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0729 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0082 | Regression_eta: 0.0006 | Regression_pt: 0.0088 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0000 | Classification_binary: 0.0729 | Regression_cos_phi: 0.0481 | Regression_energy: 0.1026 | Regression_eta: 0.0942 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0459 | LR: 6.58e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4955s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0354 | max_loss: 0.0026 | max_backward: 0.0388 | max_optimizer_step: 0.0633 | max_tensorboard_logging: 0.1357 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3454.5 | uss_MB: 2227.6 | child_rss_MB: 22740.0 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:31:13,957] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 04:31:14,027] INFO : Running validation on rank0 +rank0 [2026-11-15 04:31:14,027] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:31:14,027] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:31:14,152] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 04:31:14,256] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:31:14,355] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:31:14,423] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:31:14,488] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:31:14,551] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:31:14,617] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:31:14,682] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:31:14,795] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:31:14,905] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:31:15,013] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:31:15,080] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:31:15,145] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:31:15,210] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:31:15,276] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:31:15,341] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:31:15,449] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:31:15,456] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:31:15,458] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:31:15,459] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:31:15,460] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:31:15,464] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:31:15,464] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:31:15,466] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:31:15,468] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:31:15,469] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:31:15,471] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:31:15,473] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:31:15,473] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:31:15,475] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:31:15,477] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:31:15,479] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:31:16,891] INFO : VALIDATION | Step=30000/500000 | Train Loss=0.4761 | Valid Loss=11.8482 | Train Loss Unweighted=0.0908 | Valid Loss Unweighted=2.4545 | Stale=22 | ETA=929.8m +rank0 [2026-11-15 04:31:16,892] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:31:16,966] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:31:16,973] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:31:17,022] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:31:17,492] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:31:17,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:31:17,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:31:18,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:31:18,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:31:18,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:31:18,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:31:19,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:31:19,420] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:31:19,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:31:19,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:31:20,140] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:31:20,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:31:20,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:31:20,870] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:31:21,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:31:21,148] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:31:21,153] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:31:21,684] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:31:22,106] INFO : Plotted number of elements +rank0 [2026-11-15 04:31:23,289] INFO : Plotted jets +rank0 [2026-11-15 04:31:24,688] INFO : Plotted jet ratio +rank0 [2026-11-15 04:32:02,703] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 04:32:02,703] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:33:03,613] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 04:33:03,613] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:33:13,822] INFO : Step 31000/500000 rank0 | Train Loss: 0.6801 | Train Loss Unweighted: 0.0936 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0710 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0107 | Regression_eta: 0.0003 | Regression_pt: 0.0109 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0004 | Classification_binary: 0.0710 | Regression_cos_phi: 0.0883 | Regression_energy: 0.1332 | Regression_eta: 0.0419 | Regression_pt: 0.1399 | Regression_sin_phi: 0.2054 | LR: 6.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5201s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0359 | max_loss: 0.0026 | max_backward: 0.0385 | max_optimizer_step: 0.0778 | max_tensorboard_logging: 0.1337 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3402.5 | uss_MB: 2175.6 | child_rss_MB: 22739.8 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:33:13,823] INFO : Running validation on rank0 +rank0 [2026-11-15 04:33:13,823] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:33:13,824] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:33:13,943] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 04:33:14,055] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:33:14,159] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:33:14,226] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:33:14,291] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:33:14,355] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:33:14,420] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:33:14,486] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:33:14,597] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:33:14,698] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:33:14,796] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:33:14,864] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:33:14,929] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:33:14,994] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:33:15,061] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:33:15,126] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:33:15,234] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:33:15,241] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:33:15,243] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:33:15,245] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:33:15,247] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:33:15,249] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:33:15,250] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:33:15,252] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:33:15,254] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:33:15,254] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:33:15,256] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:33:15,258] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:33:15,259] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:33:15,261] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:33:15,263] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:33:15,265] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:33:16,666] INFO : VALIDATION | Step=31000/500000 | Train Loss=0.6801 | Valid Loss=11.9863 | Train Loss Unweighted=0.0936 | Valid Loss Unweighted=2.6181 | Stale=23 | ETA=928.1m +rank0 [2026-11-15 04:33:16,666] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:33:16,740] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:33:16,746] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:33:16,794] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:33:17,260] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:33:17,496] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:33:17,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:33:17,979] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:33:18,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:33:18,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:33:18,700] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:33:18,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:33:19,169] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:33:19,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:33:19,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:33:19,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:33:20,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:33:20,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:33:20,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:33:20,819] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:33:20,883] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 04:33:20,889] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:33:21,418] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:33:21,836] INFO : Plotted number of elements +rank0 [2026-11-15 04:33:22,997] INFO : Plotted jets +rank0 [2026-11-15 04:33:24,372] INFO : Plotted jet ratio +rank0 [2026-11-15 04:34:15,790] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-15 04:34:15,791] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:35:13,453] INFO : Step 32000/500000 rank0 | Train Loss: 1.9631 | Train Loss Unweighted: 0.1958 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1412 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0255 | Regression_eta: 0.0070 | Regression_pt: 0.0217 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0040 | Classification_binary: 0.1412 | Regression_cos_phi: 0.0691 | Regression_energy: 0.3176 | Regression_eta: 1.0349 | Regression_pt: 0.2777 | Regression_sin_phi: 0.1187 | LR: 7.19e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4949s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0361 | max_loss: 0.0026 | max_backward: 0.0392 | max_optimizer_step: 0.0652 | max_tensorboard_logging: 0.1284 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3511.6 | uss_MB: 2284.7 | child_rss_MB: 22738.0 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 04:35:13,455] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-32000.pth +rank0 [2026-11-15 04:35:13,525] INFO : Running validation on rank0 +rank0 [2026-11-15 04:35:13,525] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:35:13,525] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:35:13,646] INFO : saved batch0_epoch32000.parquet +rank0 [2026-11-15 04:35:13,753] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:35:13,859] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:35:13,927] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:35:13,992] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:35:14,056] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:35:14,122] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:35:14,188] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:35:14,301] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:35:14,411] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:35:14,520] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:35:14,587] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:35:14,653] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:35:14,718] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:35:14,784] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:35:14,850] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:35:14,963] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:35:14,969] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:35:14,972] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:35:14,974] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:35:14,976] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:35:14,977] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:35:14,979] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:35:14,979] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:35:14,982] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:35:14,983] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:35:14,985] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:35:14,986] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:35:14,988] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:35:14,990] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:35:14,992] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:35:14,994] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:35:16,424] INFO : VALIDATION | Step=32000/500000 | Train Loss=1.9631 | Valid Loss=12.7518 | Train Loss Unweighted=0.1958 | Valid Loss Unweighted=2.6648 | Stale=24 | ETA=926.4m +rank0 [2026-11-15 04:35:16,425] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:35:16,500] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:35:16,506] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:35:16,557] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:35:17,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:35:17,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:35:17,544] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:35:17,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:35:18,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:35:18,277] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:35:18,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:35:18,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:35:18,993] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:35:19,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:35:19,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:35:19,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:35:19,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:35:20,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:35:20,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:35:20,654] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:35:20,711] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:35:20,717] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:35:21,246] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:35:21,668] INFO : Plotted number of elements +rank0 [2026-11-15 04:35:22,910] INFO : Plotted jets +rank0 [2026-11-15 04:35:24,299] INFO : Plotted jet ratio +rank0 [2026-11-15 04:35:28,342] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-15 04:35:28,342] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:36:29,114] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-15 04:36:29,114] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:37:13,101] INFO : Step 33000/500000 rank0 | Train Loss: 1.2223 | Train Loss Unweighted: 0.2595 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1883 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0355 | Regression_eta: 0.0003 | Regression_pt: 0.0352 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.1883 | Regression_cos_phi: 0.0244 | Regression_energy: 0.4429 | Regression_eta: 0.0450 | Regression_pt: 0.4504 | Regression_sin_phi: 0.0712 | LR: 7.51e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5051s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0642 | max_tensorboard_logging: 0.1384 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3549.3 | uss_MB: 2322.4 | child_rss_MB: 22733.8 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:37:13,102] INFO : Running validation on rank0 +rank0 [2026-11-15 04:37:13,102] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:37:13,102] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:37:13,226] INFO : saved batch0_epoch33000.parquet +rank0 [2026-11-15 04:37:13,331] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:37:13,434] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:37:13,501] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:37:13,566] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:37:13,631] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:37:13,697] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:37:13,763] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:37:13,877] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:37:13,987] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:37:14,099] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:37:14,167] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:37:14,232] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:37:14,296] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:37:14,361] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:37:14,427] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:37:14,537] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:37:14,544] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:37:14,546] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:37:14,548] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:37:14,550] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:37:14,552] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:37:14,554] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:37:14,556] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:37:14,558] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:37:14,558] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:37:14,560] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:37:14,562] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:37:14,563] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:37:14,564] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:37:14,567] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:37:14,569] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:37:15,986] INFO : VALIDATION | Step=33000/500000 | Train Loss=1.2223 | Valid Loss=12.0923 | Train Loss Unweighted=0.2595 | Valid Loss Unweighted=2.6491 | Stale=25 | ETA=924.6m +rank0 [2026-11-15 04:37:15,986] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:37:16,059] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:37:16,065] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:37:16,115] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:37:16,586] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:37:16,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:37:17,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:37:17,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:37:17,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:37:17,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:37:18,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:37:18,274] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:37:18,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:37:18,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:37:18,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:37:19,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:37:19,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:37:19,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:37:19,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:37:20,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:37:20,230] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:37:20,235] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:37:20,758] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:37:21,176] INFO : Plotted number of elements +rank0 [2026-11-15 04:37:22,396] INFO : Plotted jets +rank0 [2026-11-15 04:37:23,749] INFO : Plotted jet ratio +rank0 [2026-11-15 04:37:41,036] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-15 04:37:41,037] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:38:41,372] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-15 04:38:41,372] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:39:12,038] INFO : Step 34000/500000 rank0 | Train Loss: 0.3936 | Train Loss Unweighted: 0.0395 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0204 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0091 | Regression_eta: 0.0003 | Regression_pt: 0.0095 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0009 | Classification_binary: 0.0204 | Regression_cos_phi: 0.0260 | Regression_energy: 0.1131 | Regression_eta: 0.0506 | Regression_pt: 0.1218 | Regression_sin_phi: 0.0609 | LR: 7.83e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4915s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0363 | max_loss: 0.0026 | max_backward: 0.0385 | max_optimizer_step: 0.0624 | max_tensorboard_logging: 0.1323 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3497.2 | uss_MB: 2270.3 | child_rss_MB: 22740.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 04:39:12,040] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-34000.pth +rank0 [2026-11-15 04:39:12,106] INFO : Running validation on rank0 +rank0 [2026-11-15 04:39:12,107] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:39:12,107] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:39:12,225] INFO : saved batch0_epoch34000.parquet +rank0 [2026-11-15 04:39:12,332] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:39:12,432] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:39:12,501] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:39:12,567] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:39:12,633] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:39:12,700] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:39:12,768] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:39:12,882] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:39:12,993] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:39:13,104] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:39:13,171] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:39:13,236] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:39:13,299] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:39:13,363] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:39:13,428] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:39:13,536] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:39:13,542] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:39:13,544] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:39:13,547] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:39:13,549] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:39:13,550] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:39:13,551] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:39:13,553] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:39:13,555] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:39:13,556] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:39:13,556] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:39:13,559] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:39:13,560] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:39:13,561] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:39:13,564] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:39:13,566] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:39:14,974] INFO : VALIDATION | Step=34000/500000 | Train Loss=0.3936 | Valid Loss=12.1877 | Train Loss Unweighted=0.0395 | Valid Loss Unweighted=2.8850 | Stale=26 | ETA=922.6m +rank0 [2026-11-15 04:39:14,975] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:39:15,049] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:39:15,056] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:39:15,107] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:39:15,594] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:39:15,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:39:16,084] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:39:16,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:39:16,563] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:39:16,796] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:39:17,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:39:17,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:39:17,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:39:17,756] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:39:17,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:39:18,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:39:18,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:39:18,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:39:18,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:39:19,156] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:39:19,219] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:39:19,226] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:39:19,774] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:39:20,185] INFO : Plotted number of elements +rank0 [2026-11-15 04:39:21,353] INFO : Plotted jets +rank0 [2026-11-15 04:39:22,713] INFO : Plotted jet ratio +rank0 [2026-11-15 04:39:53,367] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-15 04:39:53,368] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:40:53,442] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-15 04:40:53,442] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:10,620] INFO : Step 35000/500000 rank0 | Train Loss: 0.8852 | Train Loss Unweighted: 0.0856 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.0620 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0109 | Regression_eta: 0.0003 | Regression_pt: 0.0110 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0110 | Classification_binary: 0.0620 | Regression_cos_phi: 0.2261 | Regression_energy: 0.1363 | Regression_eta: 0.0515 | Regression_pt: 0.1409 | Regression_sin_phi: 0.2574 | LR: 8.16e-05 | DataLoad Time: 0.0059s | Model Forward Time: 0.5108s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0389 | max_loss: 0.0031 | max_backward: 0.0415 | max_optimizer_step: 0.0693 | max_tensorboard_logging: 0.1289 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3601.9 | uss_MB: 2375.6 | child_rss_MB: 22740.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 04:41:10,621] INFO : Running validation on rank0 +rank0 [2026-11-15 04:41:10,621] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:41:10,621] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:41:10,740] INFO : saved batch0_epoch35000.parquet +rank0 [2026-11-15 04:41:10,844] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:41:10,946] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:41:11,015] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:41:11,079] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:41:11,142] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:41:11,205] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:41:11,269] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:41:11,381] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:41:11,493] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:41:11,602] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:41:11,669] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:41:11,735] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:41:11,801] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:41:11,866] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:41:11,933] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:41:12,040] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:41:12,047] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:41:12,049] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:41:12,051] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:41:12,051] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:41:12,055] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:41:12,055] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:41:12,057] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:41:12,059] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:41:12,060] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:41:12,062] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:41:12,063] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:41:12,064] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:41:12,066] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:41:12,067] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:41:12,070] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:41:13,449] INFO : VALIDATION | Step=35000/500000 | Train Loss=0.8852 | Valid Loss=12.7311 | Train Loss Unweighted=0.0856 | Valid Loss Unweighted=2.9098 | Stale=27 | ETA=920.6m +rank0 [2026-11-15 04:41:13,449] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:41:13,522] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:41:13,528] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:41:13,577] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:41:14,049] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:41:14,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:41:14,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:41:14,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:41:15,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:41:15,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:41:15,500] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:41:15,737] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:41:15,971] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:41:16,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:41:16,453] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:41:16,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:41:16,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:41:17,176] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:41:17,411] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:41:17,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:41:17,686] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:41:17,691] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:41:18,221] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:41:18,642] INFO : Plotted number of elements +rank0 [2026-11-15 04:41:19,803] INFO : Plotted jets +rank0 [2026-11-15 04:41:21,188] INFO : Plotted jet ratio +rank0 [2026-11-15 04:42:05,242] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-15 04:42:05,242] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:43:05,531] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-15 04:43:05,532] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:43:09,353] INFO : Step 36000/500000 rank0 | Train Loss: 0.4882 | Train Loss Unweighted: 0.1337 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1173 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0080 | Regression_eta: 0.0001 | Regression_pt: 0.0079 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0000 | Classification_binary: 0.1173 | Regression_cos_phi: 0.1240 | Regression_energy: 0.1003 | Regression_eta: 0.0099 | Regression_pt: 0.1010 | Regression_sin_phi: 0.0358 | LR: 8.50e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4825s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0631 | max_tensorboard_logging: 0.1296 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3626.7 | uss_MB: 2439.8 | child_rss_MB: 22740.0 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:43:09,355] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-36000.pth +rank0 [2026-11-15 04:43:09,423] INFO : Running validation on rank0 +rank0 [2026-11-15 04:43:09,423] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:43:09,423] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:43:09,542] INFO : saved batch0_epoch36000.parquet +rank0 [2026-11-15 04:43:09,647] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:43:09,747] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:43:09,816] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:43:09,882] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:43:09,947] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:43:10,014] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:43:10,082] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:43:10,196] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:43:10,300] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:43:10,404] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:43:10,471] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:43:10,538] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:43:10,604] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:43:10,669] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:43:10,735] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:43:10,843] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:43:10,850] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:43:10,852] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:43:10,854] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:43:10,856] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:43:10,858] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:43:10,858] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:43:10,859] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:43:10,862] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:43:10,863] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:43:10,865] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:43:10,865] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:43:10,867] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:43:10,869] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:43:10,870] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:43:10,873] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:43:12,270] INFO : VALIDATION | Step=36000/500000 | Train Loss=0.4882 | Valid Loss=12.0784 | Train Loss Unweighted=0.1337 | Valid Loss Unweighted=2.8195 | Stale=28 | ETA=918.6m +rank0 [2026-11-15 04:43:12,270] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:43:12,347] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:43:12,355] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:43:12,407] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:43:12,906] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:43:13,154] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:43:13,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:43:13,635] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:43:13,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:43:14,113] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:43:14,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:43:14,582] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:43:14,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:43:15,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:43:15,321] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:43:15,562] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:43:15,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:43:16,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:43:16,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:43:16,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:43:16,546] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:43:16,551] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:43:17,090] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:43:17,502] INFO : Plotted number of elements +rank0 [2026-11-15 04:43:18,658] INFO : Plotted jets +rank0 [2026-11-15 04:43:20,045] INFO : Plotted jet ratio +rank0 [2026-11-15 04:44:17,446] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-15 04:44:17,446] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:45:08,538] INFO : Step 37000/500000 rank0 | Train Loss: 0.3716 | Train Loss Unweighted: 0.0653 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0444 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0103 | Regression_eta: 0.0001 | Regression_pt: 0.0103 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.0444 | Regression_cos_phi: 0.0172 | Regression_energy: 0.1282 | Regression_eta: 0.0203 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0296 | LR: 8.85e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4915s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0356 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0636 | max_tensorboard_logging: 0.1301 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3644.7 | uss_MB: 2472.3 | child_rss_MB: 22736.2 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 04:45:08,540] INFO : Running validation on rank0 +rank0 [2026-11-15 04:45:08,540] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:45:08,540] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:45:08,660] INFO : saved batch0_epoch37000.parquet +rank0 [2026-11-15 04:45:08,766] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:45:08,868] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:45:08,936] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:45:09,003] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:45:09,070] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:45:09,137] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:45:09,205] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:45:09,301] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:45:09,401] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:45:09,502] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:45:09,568] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:45:09,633] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:45:09,696] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:45:09,760] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:45:09,824] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:45:09,931] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:45:09,937] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:45:09,939] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:45:09,941] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:45:09,943] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:45:09,945] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:45:09,946] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:45:09,949] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:45:09,951] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:45:09,952] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:45:09,954] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:45:09,956] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:45:09,956] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:45:09,958] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:45:09,960] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:45:09,962] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:45:11,359] INFO : VALIDATION | Step=37000/500000 | Train Loss=0.3716 | Valid Loss=11.7303 | Train Loss Unweighted=0.0653 | Valid Loss Unweighted=2.7800 | Stale=29 | ETA=916.7m +rank0 [2026-11-15 04:45:11,359] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:45:11,435] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:45:11,442] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:45:11,493] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:45:11,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:45:12,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:45:12,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:45:12,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:45:12,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:45:13,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:45:13,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:45:13,675] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:45:13,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:45:14,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:45:14,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:45:14,650] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:45:14,887] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:45:15,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:45:15,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:45:15,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:45:15,636] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:45:15,641] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:45:16,182] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:45:16,592] INFO : Plotted number of elements +rank0 [2026-11-15 04:45:17,756] INFO : Plotted jets +rank0 [2026-11-15 04:45:19,128] INFO : Plotted jet ratio +rank0 [2026-11-15 04:45:29,407] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-15 04:45:29,407] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:46:30,183] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-15 04:46:30,183] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:47:07,809] INFO : Step 38000/500000 rank0 | Train Loss: 0.8592 | Train Loss Unweighted: 0.0642 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0116 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0265 | Regression_eta: 0.0002 | Regression_pt: 0.0254 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0000 | Classification_binary: 0.0116 | Regression_cos_phi: 0.0945 | Regression_energy: 0.3309 | Regression_eta: 0.0298 | Regression_pt: 0.3253 | Regression_sin_phi: 0.0671 | LR: 9.21e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4887s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0385 | max_optimizer_step: 0.0633 | max_tensorboard_logging: 0.1308 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3677.4 | uss_MB: 2505.0 | child_rss_MB: 22739.4 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:47:07,811] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-38000.pth +rank0 [2026-11-15 04:47:07,878] INFO : Running validation on rank0 +rank0 [2026-11-15 04:47:07,878] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:47:07,878] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:47:08,007] INFO : saved batch0_epoch38000.parquet +rank0 [2026-11-15 04:47:08,112] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:47:08,214] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:47:08,283] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:47:08,350] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:47:08,417] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:47:08,485] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:47:08,552] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:47:08,666] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:47:08,774] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:47:08,882] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:47:08,948] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:47:09,012] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:47:09,076] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:47:09,142] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:47:09,206] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:47:09,319] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:47:09,325] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:47:09,326] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:47:09,329] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:47:09,331] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:47:09,333] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:47:09,335] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:47:09,337] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:47:09,339] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:47:09,340] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:47:09,342] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:47:09,344] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:47:09,344] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:47:09,346] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:47:09,348] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:47:09,350] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:47:10,753] INFO : VALIDATION | Step=38000/500000 | Train Loss=0.8592 | Valid Loss=12.0072 | Train Loss Unweighted=0.0642 | Valid Loss Unweighted=3.0325 | Stale=30 | ETA=914.8m +rank0 [2026-11-15 04:47:10,753] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:47:10,828] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:47:10,834] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:47:10,884] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:47:11,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:47:11,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:47:11,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:47:12,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:47:12,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:47:12,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:47:12,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:47:13,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:47:13,337] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:47:13,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:47:13,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:47:14,056] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:47:14,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:47:14,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:47:14,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:47:14,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:47:15,045] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:47:15,051] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:47:15,585] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:47:15,990] INFO : Plotted number of elements +rank0 [2026-11-15 04:47:17,238] INFO : Plotted jets +rank0 [2026-11-15 04:47:18,607] INFO : Plotted jet ratio +rank0 [2026-11-15 04:47:18,944] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 04:47:18,944] INFO : Training completed. Total time on device 0: 75.383min diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..57df9a92a956c986bb7c2ca9740b1eb24b9854f3 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/train.log.0 @@ -0,0 +1,2964 @@ +rank0 [2026-11-15 03:31:53,265] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 03:31:53,265] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 03:31:53,266] INFO : Instantiating model +rank0 [2026-11-15 03:31:53,290] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 03:31:53,290] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 03:31:53,292] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 03:31:53,292] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 03:31:53,292] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 03:31:53,292] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 03:31:53,292] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:31:53,294] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 03:31:53,294] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:31:53,297] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 03:31:53,297] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:31:53,300] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 03:31:53,300] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:31:53,303] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 03:31:53,303] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:31:53,305] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 03:31:53,305] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:31:53,308] INFO : Backbone initialization took 0.02s +rank0 [2026-11-15 03:31:53,308] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 03:31:53,308] INFO : Initializing output DNNs +rank0 [2026-11-15 03:31:53,309] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:31:53,312] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:31:53,319] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-15 03:31:53,319] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 03:31:53,319] INFO : jagged_attention enabled=True +rank0 [2026-11-15 03:31:53,319] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 03:31:53,320] INFO : task_queries enabled=True +rank0 [2026-11-15 03:31:53,320] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 03:31:53,320] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 03:31:53,320] INFO : classification_query parameters: 256 +rank0 [2026-11-15 03:31:53,320] INFO : regression_query parameters: 256 +rank0 [2026-11-15 03:31:53,320] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 03:31:53,320] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 03:31:53,320] INFO : output_mode=elementwise +rank0 [2026-11-15 03:31:53,320] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 03:31:53,320] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 03:31:53,320] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 03:31:53,320] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 03:31:53,320] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 03:31:53,320] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 03:31:53,320] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 03:31:53,320] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 03:31:53,321] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 03:31:53,321] INFO : MLPF __init__ done +rank0 [2026-11-15 03:31:53,322] INFO : Instantiated model +rank0 [2026-11-15 03:31:53,322] INFO : Moving model to device rank=0 +rank0 [2026-11-15 03:31:53,330] INFO : Moved model to device rank=0 +rank0 [2026-11-15 03:31:53,331] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 03:31:53,449] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 03:31:53,449] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 03:31:53,449] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 03:31:53,543] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 03:31:53,543] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 03:31:53,544] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 03:31:53,544] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 03:31:53,544] INFO : Total parameters: 3910930 +rank0 [2026-11-15 03:31:53,547] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 03:31:53,552] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062 +rank0 [2026-11-15 03:31:53,552] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062 +rank0 [2026-11-15 03:31:53,552] INFO : Getting dataloaders +rank0 [2026-11-15 03:31:53,552] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:31:55,022] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:31:55,093] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 03:31:55,155] INFO : train_dataset: cld_edm_z_tautau_pf, 898 +rank0 [2026-11-15 03:31:55,197] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:31:55,243] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 03:31:55,291] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:31:55,349] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:31:55,418] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:31:55,651] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-15 03:31:55,695] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-15 03:31:55,695] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 03:31:55,695] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 03:31:55,696] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-15 03:31:55,696] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:31:55,707] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,719] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,730] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,741] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,752] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,762] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,772] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,783] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,793] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,803] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-15 03:31:55,803] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 03:31:55,803] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 03:31:55,851] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:31:55,976] INFO : Got dataloaders +rank0 [2026-11-15 03:31:55,976] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-15 03:31:55,976] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-15 03:31:55,976] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 03:31:55,986] INFO : Creating train_iterator +rank0 [2026-11-15 03:31:55,986] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:31:56,010] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:31:56,134] INFO : Created train_iterator +rank0 [2026-11-15 03:32:08,426] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.4839125296473503, 'Classification': 0.016125061931088567, 'Regression_pt': 0.11590320996940136, 'Regression_eta': 0.010036410794418771, 'Regression_sin_phi': 0.0025469883670302808, 'Regression_cos_phi': 0.004999627624201821, 'Regression_energy': 0.11904318055137991}, weights={'Classification_binary': 1.0, 'Classification': 92.02523040771484, 'Regression_pt': 12.803031921386719, 'Regression_eta': 147.8529052734375, 'Regression_sin_phi': 582.6145629882812, 'Regression_cos_phi': 296.8045959472656, 'Regression_energy': 12.465330123901367} +rank0 [2026-11-15 03:32:58,579] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 03:32:58,579] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:33:45,886] INFO : Step 1000/500000 rank0 | Train Loss: 4.5382 | Train Loss Unweighted: 1.0758 | Train Loss Components: Classification: 0.0109 | Classification_binary: 0.8709 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0969 | Regression_eta: 0.0003 | Regression_pt: 0.0963 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 1.0033 | Classification_binary: 0.8709 | Regression_cos_phi: 0.0872 | Regression_energy: 1.2082 | Regression_eta: 0.0479 | Regression_pt: 1.2329 | Regression_sin_phi: 0.0879 | LR: 2.01e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4737s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0379 | max_optimizer_step: 0.0575 | max_tensorboard_logging: 0.1365 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2604.2 | uss_MB: 1369.6 | child_rss_MB: 22943.0 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:33:45,887] INFO : Running validation on rank0 +rank0 [2026-11-15 03:33:45,887] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:33:45,888] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:33:46,057] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 03:33:46,233] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:33:46,301] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:33:46,354] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:33:46,408] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:33:46,454] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:33:46,507] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:33:46,562] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:33:46,609] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:33:46,688] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:33:46,762] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:33:46,813] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:33:46,867] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:33:46,912] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:33:46,966] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:33:47,019] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:33:47,067] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:33:47,071] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:33:47,073] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:33:47,075] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:33:47,077] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:33:47,079] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:33:47,080] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:33:47,082] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:33:47,084] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:33:47,084] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:33:47,086] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:33:47,088] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:33:47,089] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:33:47,091] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:33:47,093] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:33:47,095] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:33:48,571] INFO : VALIDATION | Step=1000/500000 | Train Loss=4.5382 | Valid Loss=8.9650 | Train Loss Unweighted=1.0758 | Valid Loss Unweighted=1.2835 | Stale=0 | ETA=936.3m +rank0 [2026-11-15 03:33:48,571] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:33:48,646] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:33:48,654] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:33:48,695] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:33:49,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:33:49,565] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:33:49,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:33:50,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:33:50,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:33:50,523] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:33:50,756] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:33:50,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:33:51,221] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:33:51,456] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:33:51,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:33:51,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:33:52,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:33:52,415] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:33:52,653] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:33:52,869] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:33:52,930] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:33:52,939] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:33:53,464] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_1000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:33:54,254] INFO : Plotted number of elements +rank0 [2026-11-15 03:33:55,465] INFO : Plotted jets +rank0 [2026-11-15 03:33:56,790] INFO : Plotted jet ratio +rank0 [2026-11-15 03:34:10,470] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 03:34:10,470] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:35:10,308] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 03:35:10,308] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:35:44,405] INFO : Step 2000/500000 rank0 | Train Loss: 3.1057 | Train Loss Unweighted: 1.0979 | Train Loss Components: Classification: 0.0031 | Classification_binary: 1.0078 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0422 | Regression_eta: 0.0003 | Regression_pt: 0.0429 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.2882 | Classification_binary: 1.0078 | Regression_cos_phi: 0.2440 | Regression_energy: 0.5255 | Regression_eta: 0.0429 | Regression_pt: 0.5499 | Regression_sin_phi: 0.4475 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4780s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0381 | max_optimizer_step: 0.0604 | max_tensorboard_logging: 0.1394 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2727.3 | uss_MB: 1500.1 | child_rss_MB: 22739.5 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:35:44,407] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 03:35:44,475] INFO : Running validation on rank0 +rank0 [2026-11-15 03:35:44,475] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:35:44,475] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:35:44,594] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 03:35:44,698] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:35:44,796] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:35:44,863] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:35:44,927] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:35:44,990] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:35:45,054] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:35:45,119] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:35:45,224] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:35:45,331] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:35:45,438] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:35:45,505] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:35:45,571] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:35:45,635] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:35:45,700] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:35:45,766] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:35:45,862] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:35:45,868] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:35:45,870] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:35:45,872] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:35:45,874] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:35:45,876] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:35:45,877] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:35:45,879] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:35:45,881] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:35:45,881] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:35:45,883] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:35:45,885] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:35:45,886] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:35:45,888] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:35:45,890] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:35:45,892] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:35:47,367] INFO : VALIDATION | Step=2000/500000 | Train Loss=3.1057 | Valid Loss=8.2246 | Train Loss Unweighted=1.0979 | Valid Loss Unweighted=1.2467 | Stale=0 | ETA=960.2m +rank0 [2026-11-15 03:35:47,368] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:35:47,440] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:35:47,452] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:35:47,501] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:35:47,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:35:48,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:35:48,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:35:48,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:35:48,918] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:35:49,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:35:49,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:35:49,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:35:49,859] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:35:50,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:35:50,329] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:35:50,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:35:50,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:35:51,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:35:51,276] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:35:51,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:35:51,552] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:35:51,557] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:35:52,074] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_2000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:35:52,486] INFO : Plotted number of elements +rank0 [2026-11-15 03:35:53,632] INFO : Plotted jets +rank0 [2026-11-15 03:35:54,953] INFO : Plotted jet ratio +rank0 [2026-11-15 03:36:21,794] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 03:36:21,794] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:37:21,794] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 03:37:21,794] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:37:42,551] INFO : Step 3000/500000 rank0 | Train Loss: 2.7435 | Train Loss Unweighted: 0.6196 | Train Loss Components: Classification: 0.0054 | Classification_binary: 0.5119 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0464 | Regression_eta: 0.0014 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.4994 | Classification_binary: 0.5119 | Regression_cos_phi: 0.1296 | Regression_energy: 0.5790 | Regression_eta: 0.2120 | Regression_pt: 0.6875 | Regression_sin_phi: 0.1240 | LR: 2.05e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4624s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0379 | max_optimizer_step: 0.0591 | max_tensorboard_logging: 0.1277 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2780.4 | uss_MB: 1553.3 | child_rss_MB: 22739.7 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:37:42,553] INFO : Running validation on rank0 +rank0 [2026-11-15 03:37:42,553] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:37:42,553] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:37:42,671] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 03:37:42,775] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:37:42,873] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:37:42,938] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:37:43,002] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:37:43,066] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:37:43,132] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:37:43,197] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:37:43,304] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:37:43,413] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:37:43,523] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:37:43,591] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:37:43,656] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:37:43,718] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:37:43,782] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:37:43,845] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:37:43,938] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:37:43,944] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:37:43,946] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:37:43,948] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:37:43,950] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:37:43,952] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:37:43,953] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:37:43,955] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:37:43,957] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:37:43,957] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:37:43,959] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:37:43,961] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:37:43,962] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:37:43,964] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:37:43,966] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:37:43,968] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:37:45,422] INFO : VALIDATION | Step=3000/500000 | Train Loss=2.7435 | Valid Loss=8.2064 | Train Loss Unweighted=0.6196 | Valid Loss Unweighted=1.2760 | Stale=0 | ETA=964.8m +rank0 [2026-11-15 03:37:45,422] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:37:45,496] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:37:45,503] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:37:45,549] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:37:45,994] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:37:46,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:37:46,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:37:46,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:37:46,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:37:47,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:37:47,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:37:47,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:37:47,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:37:48,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:37:48,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:37:48,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:37:48,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:37:49,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:37:49,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:37:49,533] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:37:49,586] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:37:49,592] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:37:50,133] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_3000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:37:50,545] INFO : Plotted number of elements +rank0 [2026-11-15 03:37:51,698] INFO : Plotted jets +rank0 [2026-11-15 03:37:53,016] INFO : Plotted jet ratio +rank0 [2026-11-15 03:38:33,010] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 03:38:33,011] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:39:32,965] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 03:39:32,965] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:39:40,434] INFO : Step 4000/500000 rank0 | Train Loss: 4.0797 | Train Loss Unweighted: 0.9729 | Train Loss Components: Classification: 0.0099 | Classification_binary: 0.8702 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0440 | Regression_eta: 0.0008 | Regression_pt: 0.0457 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.9133 | Classification_binary: 0.8702 | Regression_cos_phi: 0.2743 | Regression_energy: 0.5487 | Regression_eta: 0.1137 | Regression_pt: 0.5850 | Regression_sin_phi: 0.7745 | LR: 2.08e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4711s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0377 | max_loss: 0.0027 | max_backward: 0.0383 | max_optimizer_step: 0.0603 | max_tensorboard_logging: 0.1316 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2811.4 | uss_MB: 1584.4 | child_rss_MB: 22739.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:39:40,435] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 03:39:40,509] INFO : Running validation on rank0 +rank0 [2026-11-15 03:39:40,509] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:39:40,509] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:39:40,629] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 03:39:40,736] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:39:40,835] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:39:40,900] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:39:40,964] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:39:41,026] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:39:41,091] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:39:41,156] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:39:41,266] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:39:41,380] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:39:41,492] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:39:41,563] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:39:41,629] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:39:41,694] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:39:41,758] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:39:41,822] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:39:41,915] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:39:41,921] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:39:41,923] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:39:41,924] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:39:41,927] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:39:41,928] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:39:41,929] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:39:41,931] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:39:41,933] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:39:41,934] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:39:41,936] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:39:41,936] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:39:41,938] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:39:41,940] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:39:41,942] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:39:41,944] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:39:43,334] INFO : VALIDATION | Step=4000/500000 | Train Loss=4.0797 | Valid Loss=8.3068 | Train Loss Unweighted=0.9729 | Valid Loss Unweighted=1.2473 | Stale=1 | ETA=965.9m +rank0 [2026-11-15 03:39:43,334] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:39:43,409] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:39:43,417] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:39:43,461] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:39:43,898] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:39:44,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:39:44,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:39:44,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:39:44,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:39:45,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:39:45,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:39:45,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:39:45,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:39:46,027] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:39:46,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:39:46,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:39:46,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:39:46,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:39:47,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:39:47,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:39:47,497] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:39:47,502] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:39:48,030] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_4000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:39:48,449] INFO : Plotted number of elements +rank0 [2026-11-15 03:39:49,595] INFO : Plotted jets +rank0 [2026-11-15 03:39:50,921] INFO : Plotted jet ratio +rank0 [2026-11-15 03:40:44,231] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 03:40:44,231] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:41:38,896] INFO : Step 5000/500000 rank0 | Train Loss: 3.5022 | Train Loss Unweighted: 1.2825 | Train Loss Components: Classification: 0.0027 | Classification_binary: 1.1556 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0589 | Regression_eta: 0.0026 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.2515 | Classification_binary: 1.1556 | Regression_cos_phi: 0.1300 | Regression_energy: 0.7345 | Regression_eta: 0.3853 | Regression_pt: 0.7962 | Regression_sin_phi: 0.0491 | LR: 2.13e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4755s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0408 | max_loss: 0.0028 | max_backward: 0.0398 | max_optimizer_step: 0.0605 | max_tensorboard_logging: 0.1278 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2842.5 | uss_MB: 1615.4 | child_rss_MB: 22739.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:41:38,897] INFO : Running validation on rank0 +rank0 [2026-11-15 03:41:38,897] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:41:38,898] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:41:39,015] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 03:41:39,117] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:41:39,216] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:41:39,282] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:41:39,345] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:41:39,409] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:41:39,474] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:41:39,538] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:41:39,646] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:41:39,753] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:41:39,862] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:41:39,929] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:41:39,994] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:41:40,059] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:41:40,125] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:41:40,191] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:41:40,286] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:41:40,292] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:41:40,294] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:41:40,297] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:41:40,299] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:41:40,301] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:41:40,301] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:41:40,303] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:41:40,305] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:41:40,306] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:41:40,308] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:41:40,310] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:41:40,310] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:41:40,312] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:41:40,314] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:41:40,316] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:41:41,702] INFO : VALIDATION | Step=5000/500000 | Train Loss=3.5022 | Valid Loss=8.2481 | Train Loss Unweighted=1.2825 | Valid Loss Unweighted=1.2255 | Stale=2 | ETA=966.4m +rank0 [2026-11-15 03:41:41,702] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:41:41,774] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:41:41,781] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:41:41,826] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:41:42,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:41:42,500] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:41:42,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:41:42,979] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:41:43,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:41:43,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:41:43,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:41:43,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:41:44,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:41:44,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:41:44,646] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:41:44,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:41:45,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:41:45,368] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:41:45,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:41:45,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:41:45,875] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:41:45,881] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:41:46,402] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_5000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:41:46,818] INFO : Plotted number of elements +rank0 [2026-11-15 03:41:47,965] INFO : Plotted jets +rank0 [2026-11-15 03:41:49,297] INFO : Plotted jet ratio +rank0 [2026-11-15 03:41:55,828] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 03:41:55,828] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:42:56,497] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 03:42:56,497] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:43:37,641] INFO : Step 6000/500000 rank0 | Train Loss: 3.0654 | Train Loss Unweighted: 0.8330 | Train Loss Components: Classification: 0.0009 | Classification_binary: 0.6709 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0758 | Regression_eta: 0.0006 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0867 | Classification_binary: 0.6709 | Regression_cos_phi: 0.1455 | Regression_energy: 0.9447 | Regression_eta: 0.0877 | Regression_pt: 1.0774 | Regression_sin_phi: 0.0524 | LR: 2.19e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4628s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0344 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0594 | max_tensorboard_logging: 0.1268 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2861.6 | uss_MB: 1641.9 | child_rss_MB: 22737.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:43:37,644] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 03:43:37,713] INFO : Running validation on rank0 +rank0 [2026-11-15 03:43:37,713] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:43:37,713] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:43:37,835] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 03:43:37,941] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:43:38,040] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:43:38,107] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:43:38,173] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:43:38,238] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:43:38,304] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:43:38,370] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:43:38,479] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:43:38,588] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:43:38,696] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:43:38,763] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:43:38,827] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:43:38,890] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:43:38,954] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:43:39,018] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:43:39,113] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:43:39,118] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:43:39,121] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:43:39,123] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:43:39,125] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:43:39,127] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:43:39,127] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:43:39,129] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:43:39,131] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:43:39,132] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:43:39,133] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:43:39,135] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:43:39,136] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:43:39,138] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:43:39,140] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:43:39,142] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:43:40,539] INFO : VALIDATION | Step=6000/500000 | Train Loss=3.0654 | Valid Loss=8.3335 | Train Loss Unweighted=0.8330 | Valid Loss Unweighted=1.2626 | Stale=3 | ETA=966.8m +rank0 [2026-11-15 03:43:40,539] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:43:40,612] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:43:40,620] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:43:40,671] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:43:41,129] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:43:41,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:43:41,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:43:41,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:43:42,096] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:43:42,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:43:42,578] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:43:42,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:43:43,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:43:43,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:43:43,539] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:43:43,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:43:44,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:43:44,258] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:43:44,492] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:43:44,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:43:44,763] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:43:44,771] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:43:45,307] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_6000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:43:45,719] INFO : Plotted number of elements +rank0 [2026-11-15 03:43:46,884] INFO : Plotted jets +rank0 [2026-11-15 03:43:48,240] INFO : Plotted jet ratio +rank0 [2026-11-15 03:44:08,037] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 03:44:08,037] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:45:08,267] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 03:45:08,267] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:45:36,026] INFO : Step 7000/500000 rank0 | Train Loss: 13.2812 | Train Loss Unweighted: 1.3131 | Train Loss Components: Classification: 0.0031 | Classification_binary: 1.1142 | Regression_cos_phi: 0.0083 | Regression_energy: 0.0596 | Regression_eta: 0.0264 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.2830 | Classification_binary: 1.1142 | Regression_cos_phi: 2.4756 | Regression_energy: 0.7428 | Regression_eta: 3.9065 | Regression_pt: 1.2209 | Regression_sin_phi: 3.5381 | LR: 2.26e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4674s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0389 | max_optimizer_step: 0.0605 | max_tensorboard_logging: 0.1263 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2881.0 | uss_MB: 1654.0 | child_rss_MB: 22736.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:45:36,027] INFO : Running validation on rank0 +rank0 [2026-11-15 03:45:36,027] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:45:36,027] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:45:36,147] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 03:45:36,249] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:45:36,348] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:45:36,416] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:45:36,483] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:45:36,550] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:45:36,618] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:45:36,685] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:45:36,799] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:45:36,909] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:45:37,019] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:45:37,085] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:45:37,149] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:45:37,212] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:45:37,277] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:45:37,342] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:45:37,440] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:45:37,445] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:45:37,448] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:45:37,450] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:45:37,452] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:45:37,454] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:45:37,455] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:45:37,455] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:45:37,457] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:45:37,459] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:45:37,460] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:45:37,463] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:45:37,463] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:45:37,465] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:45:37,467] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:45:37,468] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:45:38,861] INFO : VALIDATION | Step=7000/500000 | Train Loss=13.2812 | Valid Loss=8.4009 | Train Loss Unweighted=1.3131 | Valid Loss Unweighted=1.2612 | Stale=4 | ETA=965.9m +rank0 [2026-11-15 03:45:38,861] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:45:38,934] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:45:38,940] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:45:38,985] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:45:39,478] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:45:39,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:45:39,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:45:40,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:45:40,446] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:45:40,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:45:40,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:45:41,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:45:41,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:45:41,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:45:41,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:45:42,113] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:45:42,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:45:42,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:45:42,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:45:43,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:45:43,085] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:45:43,091] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:45:43,624] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_7000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:45:44,037] INFO : Plotted number of elements +rank0 [2026-11-15 03:45:45,193] INFO : Plotted jets +rank0 [2026-11-15 03:45:46,527] INFO : Plotted jet ratio +rank0 [2026-11-15 03:46:19,614] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 03:46:19,614] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:47:19,951] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 03:47:19,951] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:47:34,390] INFO : Step 8000/500000 rank0 | Train Loss: 1.2497 | Train Loss Unweighted: 0.7576 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.7250 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0158 | Regression_eta: 0.0002 | Regression_pt: 0.0162 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0201 | Classification_binary: 0.7250 | Regression_cos_phi: 0.0439 | Regression_energy: 0.1972 | Regression_eta: 0.0223 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0338 | LR: 2.34e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4678s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0344 | max_loss: 0.0025 | max_backward: 0.0380 | max_optimizer_step: 0.0594 | max_tensorboard_logging: 0.1306 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2908.0 | uss_MB: 1681.1 | child_rss_MB: 22735.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:47:34,392] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 03:47:34,459] INFO : Running validation on rank0 +rank0 [2026-11-15 03:47:34,459] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:47:34,459] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:47:34,576] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 03:47:34,680] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:47:34,781] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:47:34,849] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:47:34,914] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:47:34,979] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:47:35,046] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:47:35,113] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:47:35,222] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:47:35,330] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:47:35,438] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:47:35,504] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:47:35,568] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:47:35,631] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:47:35,694] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:47:35,758] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:47:35,854] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:47:35,860] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:47:35,862] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:47:35,864] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:47:35,866] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:47:35,868] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:47:35,869] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:47:35,871] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:47:35,871] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:47:35,873] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:47:35,875] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:47:35,877] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:47:35,877] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:47:35,879] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:47:35,881] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:47:35,882] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:47:37,340] INFO : VALIDATION | Step=8000/500000 | Train Loss=1.2497 | Valid Loss=8.1443 | Train Loss Unweighted=0.7576 | Valid Loss Unweighted=1.2395 | Stale=0 | ETA=964.9m +rank0 [2026-11-15 03:47:37,340] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:47:37,414] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:47:37,419] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:47:37,465] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:47:37,929] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:47:38,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:47:38,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:47:38,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:47:38,878] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:47:39,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:47:39,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:47:39,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:47:39,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:47:40,077] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:47:40,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:47:40,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:47:40,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:47:41,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:47:41,251] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:47:41,473] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:47:41,527] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:47:41,532] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:47:42,058] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_8000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:47:42,463] INFO : Plotted number of elements +rank0 [2026-11-15 03:47:43,616] INFO : Plotted jets +rank0 [2026-11-15 03:47:44,931] INFO : Plotted jet ratio +rank0 [2026-11-15 03:48:31,145] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 03:48:31,145] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:49:31,115] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 03:49:31,115] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:49:32,346] INFO : Step 9000/500000 rank0 | Train Loss: 2.2076 | Train Loss Unweighted: 1.1582 | Train Loss Components: Classification: 0.0015 | Classification_binary: 1.0993 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0283 | Regression_eta: 0.0004 | Regression_pt: 0.0282 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.1416 | Classification_binary: 1.0993 | Regression_cos_phi: 0.1197 | Regression_energy: 0.3525 | Regression_eta: 0.0543 | Regression_pt: 0.3611 | Regression_sin_phi: 0.0792 | LR: 2.43e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4825s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0373 | max_loss: 0.0027 | max_backward: 0.0396 | max_optimizer_step: 0.0610 | max_tensorboard_logging: 0.1310 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2862.3 | uss_MB: 1635.4 | child_rss_MB: 22739.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 03:49:32,348] INFO : Running validation on rank0 +rank0 [2026-11-15 03:49:32,348] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:49:32,348] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:49:32,468] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 03:49:32,574] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:49:32,673] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:49:32,740] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:49:32,805] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:49:32,869] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:49:32,934] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:49:32,999] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:49:33,107] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:49:33,215] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:49:33,325] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:49:33,393] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:49:33,458] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:49:33,524] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:49:33,590] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:49:33,656] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:49:33,755] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:49:33,761] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:49:33,763] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:49:33,765] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:49:33,767] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:49:33,769] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:49:33,770] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:49:33,772] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:49:33,774] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:49:33,774] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:49:33,775] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:49:33,778] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:49:33,779] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:49:33,781] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:49:33,783] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:49:33,785] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:49:35,184] INFO : VALIDATION | Step=9000/500000 | Train Loss=2.2076 | Valid Loss=8.4169 | Train Loss Unweighted=1.1582 | Valid Loss Unweighted=1.2537 | Stale=1 | ETA=963.1m +rank0 [2026-11-15 03:49:35,184] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:49:35,257] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:49:35,260] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:49:35,306] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:49:35,749] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:49:36,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:49:36,256] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:49:36,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:49:36,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:49:36,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:49:37,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:49:37,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:49:37,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:49:37,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:49:38,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:49:38,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:49:38,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:49:38,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:49:39,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:49:39,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:49:39,430] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:49:39,436] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:49:39,968] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_9000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:49:40,389] INFO : Plotted number of elements +rank0 [2026-11-15 03:49:41,541] INFO : Plotted jets +rank0 [2026-11-15 03:49:42,928] INFO : Plotted jet ratio +rank0 [2026-11-15 03:50:42,854] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 03:50:42,854] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:51:31,199] INFO : Step 10000/500000 rank0 | Train Loss: 1.2373 | Train Loss Unweighted: 0.4699 | Train Loss Components: Classification: 0.0023 | Classification_binary: 0.4373 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0144 | Regression_eta: 0.0006 | Regression_pt: 0.0150 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.2080 | Classification_binary: 0.4373 | Regression_cos_phi: 0.0417 | Regression_energy: 0.1800 | Regression_eta: 0.0899 | Regression_pt: 0.1918 | Regression_sin_phi: 0.0886 | LR: 2.52e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4722s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0375 | max_loss: 0.0028 | max_backward: 0.0400 | max_optimizer_step: 0.0640 | max_tensorboard_logging: 0.1285 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2965.3 | uss_MB: 1738.4 | child_rss_MB: 22741.0 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:51:31,201] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 03:51:31,268] INFO : Running validation on rank0 +rank0 [2026-11-15 03:51:31,268] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:51:31,268] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:51:31,387] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 03:51:31,489] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:51:31,588] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:51:31,655] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:51:31,719] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:51:31,783] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:51:31,848] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:51:31,915] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:51:32,026] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:51:32,142] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:51:32,256] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:51:32,326] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:51:32,392] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:51:32,457] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:51:32,522] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:51:32,587] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:51:32,684] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:51:32,690] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:51:32,692] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:51:32,695] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:51:32,696] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:51:32,697] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:51:32,699] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:51:32,701] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:51:32,703] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:51:32,703] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:51:32,705] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:51:32,707] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:51:32,708] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:51:32,710] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:51:32,711] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:51:32,714] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:51:34,111] INFO : VALIDATION | Step=10000/500000 | Train Loss=1.2373 | Valid Loss=8.2623 | Train Loss Unweighted=0.4699 | Valid Loss Unweighted=1.2536 | Stale=2 | ETA=962.1m +rank0 [2026-11-15 03:51:34,111] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:51:34,183] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:51:34,199] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:51:34,245] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:51:34,698] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:51:34,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:51:35,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:51:35,437] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:51:35,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:51:35,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:51:36,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:51:36,384] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:51:36,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:51:36,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:51:37,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:51:37,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:51:37,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:51:37,817] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:51:38,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:51:38,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:51:38,327] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:51:38,332] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:51:38,863] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:51:39,299] INFO : Plotted number of elements +rank0 [2026-11-15 03:51:40,521] INFO : Plotted jets +rank0 [2026-11-15 03:51:41,904] INFO : Plotted jet ratio +rank0 [2026-11-15 03:51:54,692] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 03:51:54,692] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:52:55,212] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 03:52:55,213] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:53:30,299] INFO : Step 11000/500000 rank0 | Train Loss: 1.7160 | Train Loss Unweighted: 0.5495 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.4775 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0352 | Regression_eta: 0.0003 | Regression_pt: 0.0354 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.4775 | Regression_cos_phi: 0.1068 | Regression_energy: 0.4385 | Regression_eta: 0.0391 | Regression_pt: 0.4533 | Regression_sin_phi: 0.1466 | LR: 2.63e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4640s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0384 | max_optimizer_step: 0.0606 | max_tensorboard_logging: 0.1258 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2977.6 | uss_MB: 1750.6 | child_rss_MB: 22741.2 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:53:30,301] INFO : Running validation on rank0 +rank0 [2026-11-15 03:53:30,301] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:53:30,301] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:53:30,420] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 03:53:30,524] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:53:30,624] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:53:30,692] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:53:30,759] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:53:30,825] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:53:30,892] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:53:30,959] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:53:31,069] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:53:31,180] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:53:31,290] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:53:31,358] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:53:31,423] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:53:31,487] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:53:31,552] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:53:31,616] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:53:31,713] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:53:31,719] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:53:31,721] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:53:31,723] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:53:31,724] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:53:31,727] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:53:31,728] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:53:31,730] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:53:31,732] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:53:31,732] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:53:31,734] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:53:31,736] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:53:31,737] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:53:31,737] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:53:31,741] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:53:31,743] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:53:33,147] INFO : VALIDATION | Step=11000/500000 | Train Loss=1.7160 | Valid Loss=8.6814 | Train Loss Unweighted=0.5495 | Valid Loss Unweighted=1.2924 | Stale=3 | ETA=961.1m +rank0 [2026-11-15 03:53:33,147] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:53:33,222] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:53:33,225] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:53:33,271] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:53:33,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:53:33,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:53:34,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:53:34,450] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:53:34,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:53:34,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:53:35,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:53:35,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:53:35,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:53:35,884] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:53:36,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:53:36,368] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:53:36,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:53:36,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:53:37,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:53:37,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:53:37,348] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:53:37,354] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:53:37,892] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_11000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:53:38,306] INFO : Plotted number of elements +rank0 [2026-11-15 03:53:39,941] INFO : Plotted jets +rank0 [2026-11-15 03:53:40,875] INFO : Plotted jet ratio +rank0 [2026-11-15 03:54:07,061] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 03:54:07,061] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:55:07,640] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 03:55:07,640] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:55:29,358] INFO : Step 12000/500000 rank0 | Train Loss: 1.8046 | Train Loss Unweighted: 1.0903 | Train Loss Components: Classification: 0.0001 | Classification_binary: 1.0508 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0184 | Regression_eta: 0.0002 | Regression_pt: 0.0203 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0071 | Classification_binary: 1.0508 | Regression_cos_phi: 0.0614 | Regression_energy: 0.2295 | Regression_eta: 0.0268 | Regression_pt: 0.2595 | Regression_sin_phi: 0.1695 | LR: 2.75e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4687s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0375 | max_loss: 0.0028 | max_backward: 0.0393 | max_optimizer_step: 0.0619 | max_tensorboard_logging: 0.1292 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2994.6 | uss_MB: 1767.6 | child_rss_MB: 22740.5 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:55:29,359] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 03:55:29,427] INFO : Running validation on rank0 +rank0 [2026-11-15 03:55:29,427] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:55:29,427] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:55:29,544] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 03:55:29,648] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:55:29,747] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:55:29,815] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:55:29,880] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:55:29,943] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:55:30,008] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:55:30,074] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:55:30,184] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:55:30,286] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:55:30,387] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:55:30,453] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:55:30,520] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:55:30,586] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:55:30,652] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:55:30,719] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:55:30,819] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:55:30,825] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:55:30,827] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:55:30,829] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:55:30,831] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:55:30,833] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:55:30,834] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:55:30,834] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:55:30,838] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:55:30,838] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:55:30,840] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:55:30,842] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:55:30,843] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:55:30,845] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:55:30,847] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:55:30,849] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:55:32,256] INFO : VALIDATION | Step=12000/500000 | Train Loss=1.8046 | Valid Loss=8.4077 | Train Loss Unweighted=1.0903 | Valid Loss Unweighted=1.3178 | Stale=4 | ETA=959.9m +rank0 [2026-11-15 03:55:32,257] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:55:32,330] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:55:32,334] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:55:32,381] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:55:32,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:55:33,074] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:55:33,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:55:33,566] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:55:33,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:55:34,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:55:34,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:55:34,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:55:34,745] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:55:34,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:55:35,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:55:35,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:55:35,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:55:35,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:55:36,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:55:36,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:55:36,446] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:55:36,451] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:55:36,983] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_12000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:55:37,417] INFO : Plotted number of elements +rank0 [2026-11-15 03:55:38,652] INFO : Plotted jets +rank0 [2026-11-15 03:55:40,014] INFO : Plotted jet ratio +rank0 [2026-11-15 03:56:19,637] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 03:56:19,637] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:57:20,005] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 03:57:20,005] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:57:28,433] INFO : Step 13000/500000 rank0 | Train Loss: 1.1523 | Train Loss Unweighted: 0.4835 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.4391 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0220 | Regression_eta: 0.0001 | Regression_pt: 0.0218 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0187 | Classification_binary: 0.4391 | Regression_cos_phi: 0.0335 | Regression_energy: 0.2745 | Regression_eta: 0.0214 | Regression_pt: 0.2786 | Regression_sin_phi: 0.0865 | LR: 2.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4965s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0394 | max_loss: 0.0028 | max_backward: 0.0397 | max_optimizer_step: 0.0644 | max_tensorboard_logging: 0.1327 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3029.2 | uss_MB: 1802.2 | child_rss_MB: 22740.5 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 03:57:28,434] INFO : Running validation on rank0 +rank0 [2026-11-15 03:57:28,434] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:57:28,435] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:57:28,554] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 03:57:28,660] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:57:28,760] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:57:28,830] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:57:28,897] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:57:28,962] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:57:29,029] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:57:29,096] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:57:29,207] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:57:29,317] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:57:29,429] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:57:29,495] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:57:29,561] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:57:29,627] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:57:29,696] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:57:29,763] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:57:29,864] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:57:29,870] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:57:29,872] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:57:29,873] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:57:29,876] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:57:29,878] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:57:29,878] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:57:29,880] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:57:29,882] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:57:29,883] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:57:29,885] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:57:29,887] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:57:29,887] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:57:29,889] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:57:29,891] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:57:29,893] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:57:31,291] INFO : VALIDATION | Step=13000/500000 | Train Loss=1.1523 | Valid Loss=8.7483 | Train Loss Unweighted=0.4835 | Valid Loss Unweighted=1.3270 | Stale=5 | ETA=958.6m +rank0 [2026-11-15 03:57:31,291] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:57:31,365] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:57:31,369] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:57:31,414] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:57:31,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:57:32,098] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:57:32,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:57:32,586] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:57:32,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:57:33,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:57:33,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:57:33,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:57:33,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:57:33,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:57:34,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:57:34,477] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:57:34,717] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:57:34,952] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:57:35,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:57:35,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:57:35,457] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 03:57:35,462] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:57:35,992] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_13000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:57:36,414] INFO : Plotted number of elements +rank0 [2026-11-15 03:57:37,638] INFO : Plotted jets +rank0 [2026-11-15 03:57:38,992] INFO : Plotted jet ratio +rank0 [2026-11-15 03:58:31,732] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 03:58:31,732] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:59:27,184] INFO : Step 14000/500000 rank0 | Train Loss: 2.0174 | Train Loss Unweighted: 0.5081 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.3974 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0560 | Regression_eta: 0.0003 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0184 | Classification_binary: 0.3974 | Regression_cos_phi: 0.0326 | Regression_energy: 0.6985 | Regression_eta: 0.0417 | Regression_pt: 0.6890 | Regression_sin_phi: 0.1397 | LR: 3.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4972s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0398 | max_optimizer_step: 0.0611 | max_tensorboard_logging: 0.1344 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3056.3 | uss_MB: 1829.3 | child_rss_MB: 22738.7 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 03:59:27,185] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 03:59:27,250] INFO : Running validation on rank0 +rank0 [2026-11-15 03:59:27,251] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:59:27,251] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:59:27,370] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 03:59:27,474] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:59:27,577] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:59:27,645] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:59:27,710] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:59:27,774] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:59:27,839] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:59:27,905] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:59:28,020] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:59:28,130] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:59:28,240] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:59:28,309] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:59:28,376] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:59:28,441] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:59:28,507] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:59:28,573] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:59:28,677] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:59:28,683] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:59:28,685] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:59:28,687] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:59:28,689] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:59:28,691] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:59:28,692] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:59:28,694] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:59:28,696] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:59:28,697] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:59:28,698] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:59:28,699] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:59:28,701] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:59:28,703] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:59:28,705] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:59:28,707] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:59:30,112] INFO : VALIDATION | Step=14000/500000 | Train Loss=2.0174 | Valid Loss=9.0274 | Train Loss Unweighted=0.5081 | Valid Loss Unweighted=1.3847 | Stale=6 | ETA=957.0m +rank0 [2026-11-15 03:59:30,112] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:59:30,186] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 03:59:30,191] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 03:59:30,238] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:59:30,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 03:59:30,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 03:59:31,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 03:59:31,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 03:59:31,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 03:59:31,923] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 03:59:32,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 03:59:32,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 03:59:32,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 03:59:32,874] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 03:59:33,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 03:59:33,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 03:59:33,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 03:59:33,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 03:59:34,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 03:59:34,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 03:59:34,342] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:59:34,348] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:59:34,877] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_14000/cld_edm_z_tautau_pf +rank0 [2026-11-15 03:59:35,294] INFO : Plotted number of elements +rank0 [2026-11-15 03:59:36,508] INFO : Plotted jets +rank0 [2026-11-15 03:59:37,866] INFO : Plotted jet ratio +rank0 [2026-11-15 03:59:43,529] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 03:59:43,529] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:00:44,202] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 04:00:44,202] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:01:26,000] INFO : Step 15000/500000 rank0 | Train Loss: 2.1749 | Train Loss Unweighted: 0.7710 | Train Loss Components: Classification: 0.0025 | Classification_binary: 0.7017 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0335 | Regression_eta: 0.0004 | Regression_pt: 0.0319 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.2343 | Classification_binary: 0.7017 | Regression_cos_phi: 0.1773 | Regression_energy: 0.4174 | Regression_eta: 0.0642 | Regression_pt: 0.4090 | Regression_sin_phi: 0.1711 | LR: 3.17e-05 | DataLoad Time: 0.0004s | Model Forward Time: 0.4669s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0344 | max_loss: 0.0025 | max_backward: 0.0380 | max_optimizer_step: 0.0605 | max_tensorboard_logging: 0.1270 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3079.3 | uss_MB: 1852.3 | child_rss_MB: 22737.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:01:26,001] INFO : Running validation on rank0 +rank0 [2026-11-15 04:01:26,002] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:01:26,002] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:01:26,125] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 04:01:26,233] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:01:26,335] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:01:26,402] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:01:26,468] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:01:26,534] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:01:26,601] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:01:26,668] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:01:26,780] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:01:26,886] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:01:26,993] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:01:27,060] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:01:27,124] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:01:27,187] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:01:27,250] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:01:27,314] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:01:27,413] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:01:27,419] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:01:27,421] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:01:27,423] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:01:27,425] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:01:27,427] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:01:27,428] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:01:27,428] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:01:27,430] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:01:27,432] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:01:27,434] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:01:27,436] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:01:27,437] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:01:27,437] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:01:27,440] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:01:27,443] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:01:28,826] INFO : VALIDATION | Step=15000/500000 | Train Loss=2.1749 | Valid Loss=8.9581 | Train Loss Unweighted=0.7710 | Valid Loss Unweighted=1.4120 | Stale=7 | ETA=955.4m +rank0 [2026-11-15 04:01:28,826] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:01:28,898] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:01:28,909] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:01:28,956] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:01:29,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:01:29,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:01:29,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:01:30,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:01:30,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:01:30,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:01:30,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:01:31,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:01:31,335] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:01:31,574] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:01:31,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:01:32,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:01:32,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:01:32,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:01:32,751] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:01:32,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:01:33,049] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:01:33,058] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:01:33,598] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_15000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:01:34,005] INFO : Plotted number of elements +rank0 [2026-11-15 04:01:35,259] INFO : Plotted jets +rank0 [2026-11-15 04:01:36,624] INFO : Plotted jet ratio +rank0 [2026-11-15 04:01:55,726] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 04:01:55,726] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:02:56,506] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 04:02:56,506] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:03:25,590] INFO : Step 16000/500000 rank0 | Train Loss: 0.8304 | Train Loss Unweighted: 0.4566 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.4323 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0118 | Regression_eta: 0.0001 | Regression_pt: 0.0118 | Regression_sin_phi: 0.0000 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0301 | Classification_binary: 0.4323 | Regression_cos_phi: 0.0248 | Regression_energy: 0.1476 | Regression_eta: 0.0186 | Regression_pt: 0.1508 | Regression_sin_phi: 0.0261 | LR: 3.33e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4829s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0378 | max_optimizer_step: 0.0605 | max_tensorboard_logging: 0.1344 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3102.5 | uss_MB: 1875.6 | child_rss_MB: 22737.4 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:03:25,592] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 04:03:25,660] INFO : Running validation on rank0 +rank0 [2026-11-15 04:03:25,660] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:03:25,660] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:03:25,789] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 04:03:25,895] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:03:25,998] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:03:26,066] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:03:26,133] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:03:26,199] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:03:26,266] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:03:26,332] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:03:26,441] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:03:26,549] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:03:26,657] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:03:26,724] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:03:26,790] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:03:26,853] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:03:26,917] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:03:26,982] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:03:27,081] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:03:27,087] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:03:27,089] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:03:27,091] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:03:27,093] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:03:27,094] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:03:27,095] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:03:27,097] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:03:27,099] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:03:27,100] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:03:27,102] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:03:27,104] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:03:27,104] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:03:27,106] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:03:27,108] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:03:27,110] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:03:28,517] INFO : VALIDATION | Step=16000/500000 | Train Loss=0.8304 | Valid Loss=9.1480 | Train Loss Unweighted=0.4566 | Valid Loss Unweighted=1.3879 | Stale=8 | ETA=954.2m +rank0 [2026-11-15 04:03:28,517] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:03:28,601] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:03:28,604] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:03:28,646] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:03:29,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:03:29,330] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:03:29,566] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:03:29,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:03:30,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:03:30,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:03:30,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:03:30,752] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:03:30,991] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:03:31,229] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:03:31,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:03:31,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:03:31,928] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:03:32,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:03:32,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:03:32,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:03:32,678] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 04:03:32,683] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:03:33,211] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_16000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:03:33,617] INFO : Plotted number of elements +rank0 [2026-11-15 04:03:34,865] INFO : Plotted jets +rank0 [2026-11-15 04:03:36,249] INFO : Plotted jet ratio +rank0 [2026-11-15 04:04:08,756] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 04:04:08,756] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:05:09,552] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 04:05:09,553] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:05:25,138] INFO : Step 17000/500000 rank0 | Train Loss: 0.7870 | Train Loss Unweighted: 0.3152 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.2858 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0145 | Regression_eta: 0.0002 | Regression_pt: 0.0143 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0221 | Classification_binary: 0.2858 | Regression_cos_phi: 0.0315 | Regression_energy: 0.1803 | Regression_eta: 0.0285 | Regression_pt: 0.1836 | Regression_sin_phi: 0.0552 | LR: 3.51e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4971s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0358 | max_loss: 0.0027 | max_backward: 0.0389 | max_optimizer_step: 0.0652 | max_tensorboard_logging: 0.1343 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3126.6 | uss_MB: 1899.6 | child_rss_MB: 22762.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:05:25,140] INFO : Running validation on rank0 +rank0 [2026-11-15 04:05:25,140] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:05:25,140] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:05:25,261] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 04:05:25,369] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:05:25,470] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:05:25,537] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:05:25,602] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:05:25,667] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:05:25,734] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:05:25,800] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:05:25,908] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:05:26,017] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:05:26,126] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:05:26,198] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:05:26,267] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:05:26,334] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:05:26,401] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:05:26,470] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:05:26,575] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:05:26,581] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:05:26,583] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:05:26,585] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:05:26,587] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:05:26,589] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:05:26,590] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:05:26,590] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:05:26,593] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:05:26,594] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:05:26,596] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:05:26,598] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:05:26,599] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:05:26,601] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:05:26,603] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:05:26,605] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:05:28,032] INFO : VALIDATION | Step=17000/500000 | Train Loss=0.7870 | Valid Loss=9.5913 | Train Loss Unweighted=0.3152 | Valid Loss Unweighted=1.4813 | Stale=9 | ETA=952.8m +rank0 [2026-11-15 04:05:28,033] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:05:28,107] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:05:28,110] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:05:28,158] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:05:28,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:05:28,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:05:29,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:05:29,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:05:29,582] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:05:29,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:05:30,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:05:30,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:05:30,533] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:05:30,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:05:31,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:05:31,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:05:31,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:05:31,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:05:31,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:05:32,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:05:32,248] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:05:32,256] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:05:32,783] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_17000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:05:33,202] INFO : Plotted number of elements +rank0 [2026-11-15 04:05:34,371] INFO : Plotted jets +rank0 [2026-11-15 04:05:35,734] INFO : Plotted jet ratio +rank0 [2026-11-15 04:06:21,838] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 04:06:21,838] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:07:22,887] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 04:07:22,887] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:07:24,988] INFO : Step 18000/500000 rank0 | Train Loss: 0.8505 | Train Loss Unweighted: 0.1724 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1249 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0234 | Regression_eta: 0.0002 | Regression_pt: 0.0237 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0023 | Classification_binary: 0.1249 | Regression_cos_phi: 0.0582 | Regression_energy: 0.2914 | Regression_eta: 0.0271 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0437 | LR: 3.69e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4833s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0026 | max_backward: 0.0388 | max_optimizer_step: 0.0626 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3155.6 | uss_MB: 1928.7 | child_rss_MB: 22767.3 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 04:07:24,990] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 04:07:25,057] INFO : Running validation on rank0 +rank0 [2026-11-15 04:07:25,057] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:07:25,057] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:07:25,176] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 04:07:25,282] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:07:25,383] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:07:25,451] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:07:25,517] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:07:25,582] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:07:25,647] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:07:25,712] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:07:25,821] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:07:25,930] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:07:26,040] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:07:26,108] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:07:26,175] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:07:26,241] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:07:26,309] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:07:26,376] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:07:26,478] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:07:26,484] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:07:26,486] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:07:26,489] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:07:26,490] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:07:26,492] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:07:26,493] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:07:26,495] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:07:26,497] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:07:26,498] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:07:26,500] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:07:26,501] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:07:26,502] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:07:26,504] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:07:26,506] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:07:26,508] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:07:27,927] INFO : VALIDATION | Step=18000/500000 | Train Loss=0.8505 | Valid Loss=9.2889 | Train Loss Unweighted=0.1724 | Valid Loss Unweighted=1.4915 | Stale=10 | ETA=951.5m +rank0 [2026-11-15 04:07:27,928] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:07:28,002] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:07:28,007] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:07:28,055] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:07:28,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:07:28,764] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:07:29,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:07:29,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:07:29,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:07:29,745] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:07:29,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:07:30,227] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:07:30,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:07:30,698] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:07:30,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:07:31,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:07:31,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:07:31,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:07:31,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:07:32,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:07:32,180] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:07:32,193] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:07:32,721] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_18000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:07:33,144] INFO : Plotted number of elements +rank0 [2026-11-15 04:07:34,381] INFO : Plotted jets +rank0 [2026-11-15 04:07:35,758] INFO : Plotted jet ratio +rank0 [2026-11-15 04:08:34,942] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 04:08:34,942] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:09:24,036] INFO : Step 19000/500000 rank0 | Train Loss: 1.3460 | Train Loss Unweighted: 0.5704 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.5411 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0137 | Regression_eta: 0.0003 | Regression_pt: 0.0140 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.5411 | Regression_cos_phi: 0.1455 | Regression_energy: 0.1706 | Regression_eta: 0.0411 | Regression_pt: 0.1792 | Regression_sin_phi: 0.2226 | LR: 3.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4742s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0345 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0603 | max_tensorboard_logging: 0.1305 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3186.5 | uss_MB: 1959.5 | child_rss_MB: 22738.7 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 04:09:24,038] INFO : Running validation on rank0 +rank0 [2026-11-15 04:09:24,038] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:09:24,038] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:09:24,167] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 04:09:24,270] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:09:24,372] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:09:24,441] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:09:24,508] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:09:24,575] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:09:24,642] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:09:24,708] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:09:24,816] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:09:24,925] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:09:25,033] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:09:25,099] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:09:25,163] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:09:25,226] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:09:25,291] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:09:25,356] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:09:25,454] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:09:25,460] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:09:25,462] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:09:25,465] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:09:25,465] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:09:25,468] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:09:25,469] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:09:25,471] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:09:25,473] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:09:25,474] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:09:25,475] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:09:25,476] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:09:25,478] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:09:25,480] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:09:25,482] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:09:25,484] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:09:26,872] INFO : VALIDATION | Step=19000/500000 | Train Loss=1.3460 | Valid Loss=10.1494 | Train Loss Unweighted=0.5704 | Valid Loss Unweighted=1.5967 | Stale=11 | ETA=949.7m +rank0 [2026-11-15 04:09:26,872] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:09:26,947] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:09:26,962] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:09:27,010] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:09:27,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:09:27,725] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:09:27,963] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:09:28,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:09:28,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:09:28,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:09:28,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:09:29,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:09:29,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:09:29,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:09:29,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:09:30,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:09:30,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:09:30,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:09:30,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:09:31,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:09:31,085] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 04:09:31,094] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:09:31,631] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_19000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:09:32,041] INFO : Plotted number of elements +rank0 [2026-11-15 04:09:33,207] INFO : Plotted jets +rank0 [2026-11-15 04:09:34,555] INFO : Plotted jet ratio +rank0 [2026-11-15 04:09:46,491] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 04:09:46,491] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:10:46,913] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 04:10:46,913] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:11:23,144] INFO : Step 20000/500000 rank0 | Train Loss: 0.7822 | Train Loss Unweighted: 0.2006 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1690 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0158 | Regression_eta: 0.0005 | Regression_pt: 0.0148 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0023 | Classification_binary: 0.1690 | Regression_cos_phi: 0.0572 | Regression_energy: 0.1968 | Regression_eta: 0.0792 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0877 | LR: 4.07e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4959s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0027 | max_backward: 0.0396 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.1355 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3133.3 | uss_MB: 1906.4 | child_rss_MB: 22740.6 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:11:23,146] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 04:11:23,214] INFO : Running validation on rank0 +rank0 [2026-11-15 04:11:23,214] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:11:23,214] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:11:23,331] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 04:11:23,437] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:11:23,537] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:11:23,604] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:11:23,668] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:11:23,732] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:11:23,799] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:11:23,865] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:11:23,974] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:11:24,086] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:11:24,196] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:11:24,264] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:11:24,330] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:11:24,394] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:11:24,458] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:11:24,522] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:11:24,624] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:11:24,630] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:11:24,632] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:11:24,634] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:11:24,636] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:11:24,638] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:11:24,639] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:11:24,640] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:11:24,643] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:11:24,643] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:11:24,645] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:11:24,647] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:11:24,648] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:11:24,650] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:11:24,650] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:11:24,654] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:11:26,041] INFO : VALIDATION | Step=20000/500000 | Train Loss=0.7822 | Valid Loss=9.7266 | Train Loss Unweighted=0.2006 | Valid Loss Unweighted=1.5992 | Stale=12 | ETA=948.0m +rank0 [2026-11-15 04:11:26,042] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:11:26,116] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:11:26,125] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:11:26,172] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:11:26,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:11:26,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:11:27,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:11:27,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:11:27,581] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:11:27,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:11:28,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:11:28,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:11:28,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:11:28,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:11:29,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:11:29,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:11:29,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:11:29,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:11:29,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:11:30,176] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:11:30,237] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 04:11:30,242] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:11:30,774] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:11:31,195] INFO : Plotted number of elements +rank0 [2026-11-15 04:11:32,348] INFO : Plotted jets +rank0 [2026-11-15 04:11:33,722] INFO : Plotted jet ratio +rank0 [2026-11-15 04:11:58,936] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 04:11:58,937] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:12:59,591] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 04:12:59,591] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:13:22,136] INFO : Step 21000/500000 rank0 | Train Loss: 2.7784 | Train Loss Unweighted: 0.3121 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.1191 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0974 | Regression_eta: 0.0002 | Regression_pt: 0.0947 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0288 | Classification_binary: 0.1191 | Regression_cos_phi: 0.0292 | Regression_energy: 1.2142 | Regression_eta: 0.0252 | Regression_pt: 1.2127 | Regression_sin_phi: 0.1493 | LR: 4.28e-05 | DataLoad Time: 0.0004s | Model Forward Time: 0.4905s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0026 | max_backward: 0.0439 | max_optimizer_step: 0.0652 | max_tensorboard_logging: 0.1293 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3233.2 | uss_MB: 2006.2 | child_rss_MB: 22739.8 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:13:22,137] INFO : Running validation on rank0 +rank0 [2026-11-15 04:13:22,137] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:13:22,137] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:13:22,258] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 04:13:22,363] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:13:22,465] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:13:22,532] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:13:22,597] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:13:22,662] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:13:22,728] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:13:22,794] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:13:22,907] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:13:23,017] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:13:23,127] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:13:23,196] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:13:23,262] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:13:23,326] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:13:23,393] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:13:23,460] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:13:23,571] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:13:23,578] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:13:23,579] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:13:23,583] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:13:23,585] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:13:23,585] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:13:23,587] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:13:23,589] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:13:23,591] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:13:23,592] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:13:23,594] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:13:23,595] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:13:23,597] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:13:23,599] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:13:23,601] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:13:23,603] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:13:25,029] INFO : VALIDATION | Step=21000/500000 | Train Loss=2.7784 | Valid Loss=9.9692 | Train Loss Unweighted=0.3121 | Valid Loss Unweighted=1.6642 | Stale=13 | ETA=946.2m +rank0 [2026-11-15 04:13:25,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:13:25,102] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:13:25,105] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:13:25,153] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:13:25,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:13:25,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:13:26,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:13:26,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:13:26,600] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:13:26,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:13:27,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:13:27,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:13:27,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:13:27,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:13:28,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:13:28,267] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:13:28,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:13:28,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:13:28,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:13:29,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:13:29,272] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:13:29,277] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:13:29,806] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_21000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:13:30,226] INFO : Plotted number of elements +rank0 [2026-11-15 04:13:31,391] INFO : Plotted jets +rank0 [2026-11-15 04:13:32,769] INFO : Plotted jet ratio +rank0 [2026-11-15 04:14:11,671] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 04:14:11,671] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:15:12,629] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 04:15:12,630] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:15:21,907] INFO : Step 22000/500000 rank0 | Train Loss: 1.2712 | Train Loss Unweighted: 0.4533 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.4048 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0245 | Regression_eta: 0.0003 | Regression_pt: 0.0231 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0061 | Classification_binary: 0.4048 | Regression_cos_phi: 0.1282 | Regression_energy: 0.3048 | Regression_eta: 0.0456 | Regression_pt: 0.2953 | Regression_sin_phi: 0.0864 | LR: 4.50e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4917s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0026 | max_backward: 0.0384 | max_optimizer_step: 0.0649 | max_tensorboard_logging: 0.1322 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3259.7 | uss_MB: 2032.7 | child_rss_MB: 22739.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:15:21,909] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 04:15:21,978] INFO : Running validation on rank0 +rank0 [2026-11-15 04:15:21,978] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:15:21,978] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:15:22,099] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 04:15:22,203] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:15:22,303] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:15:22,371] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:15:22,437] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:15:22,501] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:15:22,566] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:15:22,631] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:15:22,743] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:15:22,854] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:15:22,963] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:15:23,033] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:15:23,099] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:15:23,165] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:15:23,231] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:15:23,298] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:15:23,400] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:15:23,406] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:15:23,408] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:15:23,411] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:15:23,411] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:15:23,414] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:15:23,415] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:15:23,417] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:15:23,419] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:15:23,420] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:15:23,422] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:15:23,424] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:15:23,424] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:15:23,426] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:15:23,428] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:15:23,431] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:15:24,855] INFO : VALIDATION | Step=22000/500000 | Train Loss=1.2712 | Valid Loss=10.0428 | Train Loss Unweighted=0.4533 | Valid Loss Unweighted=1.7843 | Stale=14 | ETA=944.7m +rank0 [2026-11-15 04:15:24,856] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:15:24,931] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:15:24,938] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:15:24,986] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:15:25,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:15:25,700] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:15:25,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:15:26,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:15:26,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:15:26,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:15:26,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:15:27,153] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:15:27,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:15:27,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:15:27,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:15:28,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:15:28,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:15:28,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:15:28,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:15:29,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:15:29,095] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:15:29,101] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:15:29,627] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_22000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:15:30,048] INFO : Plotted number of elements +rank0 [2026-11-15 04:15:31,206] INFO : Plotted jets +rank0 [2026-11-15 04:15:32,553] INFO : Plotted jet ratio +rank0 [2026-11-15 04:16:24,530] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 04:16:24,530] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:17:20,872] INFO : Step 23000/500000 rank0 | Train Loss: 11.8012 | Train Loss Unweighted: 0.8686 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.7498 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0424 | Regression_eta: 0.0207 | Regression_pt: 0.0417 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.7498 | Regression_cos_phi: 0.9199 | Regression_energy: 0.5288 | Regression_eta: 3.0652 | Regression_pt: 0.5340 | Regression_sin_phi: 5.9522 | LR: 4.73e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4750s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0382 | max_optimizer_step: 0.0614 | max_tensorboard_logging: 0.1275 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3213.1 | uss_MB: 1986.2 | child_rss_MB: 22738.9 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 04:17:20,874] INFO : Running validation on rank0 +rank0 [2026-11-15 04:17:20,874] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:17:20,874] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:17:20,994] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 04:17:21,100] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:17:21,200] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:17:21,269] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:17:21,336] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:17:21,401] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:17:21,468] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:17:21,535] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:17:21,646] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:17:21,750] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:17:21,852] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:17:21,920] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:17:21,983] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:17:22,047] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:17:22,113] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:17:22,177] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:17:22,282] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:17:22,288] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:17:22,290] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:17:22,293] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:17:22,295] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:17:22,296] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:17:22,297] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:17:22,299] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:17:22,301] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:17:22,302] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:17:22,304] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:17:22,304] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:17:22,306] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:17:22,308] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:17:22,310] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:17:22,312] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:17:23,716] INFO : VALIDATION | Step=23000/500000 | Train Loss=11.8012 | Valid Loss=10.5621 | Train Loss Unweighted=0.8686 | Valid Loss Unweighted=1.9085 | Stale=15 | ETA=942.8m +rank0 [2026-11-15 04:17:23,716] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:17:23,790] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:17:23,804] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:17:23,852] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:17:24,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:17:24,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:17:24,805] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:17:25,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:17:25,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:17:25,524] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:17:25,757] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:17:25,993] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:17:26,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:17:26,473] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:17:26,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:17:26,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:17:27,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:17:27,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:17:27,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:17:27,874] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:17:27,930] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 04:17:27,938] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:17:28,477] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_23000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:17:28,893] INFO : Plotted number of elements +rank0 [2026-11-15 04:17:30,067] INFO : Plotted jets +rank0 [2026-11-15 04:17:31,434] INFO : Plotted jet ratio +rank0 [2026-11-15 04:17:36,278] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 04:17:36,278] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:18:36,459] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 04:18:36,460] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:19:19,435] INFO : Step 24000/500000 rank0 | Train Loss: 1.5245 | Train Loss Unweighted: 0.3563 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.3078 | Regression_cos_phi: 0.0011 | Regression_energy: 0.0250 | Regression_eta: 0.0008 | Regression_pt: 0.0212 | Regression_sin_phi: 0.0003 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0013 | Classification_binary: 0.3078 | Regression_cos_phi: 0.3137 | Regression_energy: 0.3121 | Regression_eta: 0.1165 | Regression_pt: 0.2716 | Regression_sin_phi: 0.2015 | LR: 4.97e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5212s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0377 | max_loss: 0.0028 | max_backward: 0.0394 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1376 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3308.9 | uss_MB: 2082.0 | child_rss_MB: 22737.5 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 04:19:19,437] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 04:19:19,506] INFO : Running validation on rank0 +rank0 [2026-11-15 04:19:19,506] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:19:19,506] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:19:19,626] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 04:19:19,729] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:19:19,828] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:19:19,897] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:19:19,964] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:19:20,030] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:19:20,098] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:19:20,164] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:19:20,278] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:19:20,387] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:19:20,493] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:19:20,560] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:19:20,626] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:19:20,690] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:19:20,756] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:19:20,824] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:19:20,930] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:19:20,936] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:19:20,939] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:19:20,939] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:19:20,941] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:19:20,945] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:19:20,945] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:19:20,947] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:19:20,949] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:19:20,950] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:19:20,952] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:19:20,953] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:19:20,954] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:19:20,955] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:19:20,957] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:19:20,960] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:19:22,352] INFO : VALIDATION | Step=24000/500000 | Train Loss=1.5245 | Valid Loss=10.7359 | Train Loss Unweighted=0.3563 | Valid Loss Unweighted=2.1115 | Stale=16 | ETA=940.9m +rank0 [2026-11-15 04:19:22,353] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:19:22,426] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:19:22,433] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:19:22,481] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:19:22,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:19:23,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:19:23,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:19:23,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:19:23,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:19:24,156] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:19:24,394] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:19:24,630] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:19:24,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:19:25,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:19:25,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:19:25,578] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:19:25,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:19:26,057] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:19:26,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:19:26,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:19:26,572] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 04:19:26,577] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:19:27,101] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_24000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:19:27,517] INFO : Plotted number of elements +rank0 [2026-11-15 04:19:28,671] INFO : Plotted jets +rank0 [2026-11-15 04:19:30,030] INFO : Plotted jet ratio +rank0 [2026-11-15 04:19:48,169] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 04:19:48,169] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:20:48,547] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 04:20:48,547] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:21:18,063] INFO : Step 25000/500000 rank0 | Train Loss: 1.9117 | Train Loss Unweighted: 0.7077 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.6707 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0201 | Regression_eta: 0.0037 | Regression_pt: 0.0126 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0003 | Classification_binary: 0.6707 | Regression_cos_phi: 0.0518 | Regression_energy: 0.2508 | Regression_eta: 0.5486 | Regression_pt: 0.1613 | Regression_sin_phi: 0.2283 | LR: 5.22e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4771s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0358 | max_loss: 0.0026 | max_backward: 0.0389 | max_optimizer_step: 0.0626 | max_tensorboard_logging: 0.1256 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3328.1 | uss_MB: 2101.2 | child_rss_MB: 22730.6 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 04:21:18,064] INFO : Running validation on rank0 +rank0 [2026-11-15 04:21:18,065] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:21:18,065] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:21:18,186] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 04:21:18,290] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:21:18,391] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:21:18,460] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:21:18,524] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:21:18,588] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:21:18,657] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:21:18,725] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:21:18,839] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:21:18,952] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:21:19,062] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:21:19,132] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:21:19,199] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:21:19,265] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:21:19,331] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:21:19,398] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:21:19,502] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:21:19,508] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:21:19,509] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:21:19,512] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:21:19,514] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:21:19,516] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:21:19,517] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:21:19,517] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:21:19,519] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:21:19,521] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:21:19,523] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:21:19,527] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:21:19,527] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:21:19,529] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:21:19,531] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:21:19,534] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:21:20,955] INFO : VALIDATION | Step=25000/500000 | Train Loss=1.9117 | Valid Loss=10.4990 | Train Loss Unweighted=0.7077 | Valid Loss Unweighted=1.9100 | Stale=17 | ETA=938.9m +rank0 [2026-11-15 04:21:20,955] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:21:21,030] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:21:21,034] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:21:21,082] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:21:21,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:21:21,789] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:21:22,053] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:21:22,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:21:22,539] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:21:22,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:21:23,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:21:23,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:21:23,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:21:23,733] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:21:23,979] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:21:24,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:21:24,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:21:24,700] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:21:24,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:21:25,149] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:21:25,212] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:21:25,220] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:21:25,752] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_25000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:21:26,177] INFO : Plotted number of elements +rank0 [2026-11-15 04:21:27,341] INFO : Plotted jets +rank0 [2026-11-15 04:21:28,725] INFO : Plotted jet ratio +rank0 [2026-11-15 04:22:00,493] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 04:22:00,494] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:23:01,359] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 04:23:01,359] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:23:17,763] INFO : Step 26000/500000 rank0 | Train Loss: 1.1273 | Train Loss Unweighted: 0.5497 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.5133 | Regression_cos_phi: 0.0000 | Regression_energy: 0.0182 | Regression_eta: 0.0008 | Regression_pt: 0.0173 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0007 | Classification_binary: 0.5133 | Regression_cos_phi: 0.0120 | Regression_energy: 0.2264 | Regression_eta: 0.1188 | Regression_pt: 0.2214 | Regression_sin_phi: 0.0348 | LR: 5.47e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.4805s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0028 | max_backward: 0.0385 | max_optimizer_step: 0.0636 | max_tensorboard_logging: 0.1273 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3346.3 | uss_MB: 2119.3 | child_rss_MB: 22740.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 04:23:17,765] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 04:23:17,832] INFO : Running validation on rank0 +rank0 [2026-11-15 04:23:17,832] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:23:17,832] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:23:17,954] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 04:23:18,059] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:23:18,160] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:23:18,228] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:23:18,294] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:23:18,359] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:23:18,425] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:23:18,493] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:23:18,606] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:23:18,710] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:23:18,812] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:23:18,880] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:23:18,946] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:23:19,012] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:23:19,080] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:23:19,145] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:23:19,252] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:23:19,259] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:23:19,261] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:23:19,263] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:23:19,266] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:23:19,268] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:23:19,269] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:23:19,271] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:23:19,273] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:23:19,274] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:23:19,274] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:23:19,277] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:23:19,278] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:23:19,279] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:23:19,282] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:23:19,284] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:23:20,699] INFO : VALIDATION | Step=26000/500000 | Train Loss=1.1273 | Valid Loss=11.0044 | Train Loss Unweighted=0.5497 | Valid Loss Unweighted=2.1101 | Stale=18 | ETA=937.3m +rank0 [2026-11-15 04:23:20,699] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:23:20,773] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:23:20,780] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:23:20,829] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:23:21,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:23:21,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:23:21,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:23:22,067] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:23:22,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:23:22,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:23:22,783] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:23:23,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:23:23,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:23:23,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:23:23,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:23:23,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:23:24,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:23:24,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:23:24,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:23:24,903] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:23:24,964] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:23:24,970] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:23:25,510] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_26000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:23:25,930] INFO : Plotted number of elements +rank0 [2026-11-15 04:23:27,083] INFO : Plotted jets +rank0 [2026-11-15 04:23:28,439] INFO : Plotted jet ratio +rank0 [2026-11-15 04:24:13,136] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 04:24:13,136] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:25:13,603] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 04:25:13,603] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:25:16,610] INFO : Step 27000/500000 rank0 | Train Loss: 0.6375 | Train Loss Unweighted: 0.0663 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.0387 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0130 | Regression_eta: 0.0002 | Regression_pt: 0.0135 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0351 | Classification_binary: 0.0387 | Regression_cos_phi: 0.0597 | Regression_energy: 0.1625 | Regression_eta: 0.0340 | Regression_pt: 0.1728 | Regression_sin_phi: 0.1346 | LR: 5.74e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4892s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0388 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.1288 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3381.2 | uss_MB: 2154.3 | child_rss_MB: 22739.3 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:25:16,611] INFO : Running validation on rank0 +rank0 [2026-11-15 04:25:16,611] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:25:16,611] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:25:16,732] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 04:25:16,835] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:25:16,935] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:25:17,004] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:25:17,070] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:25:17,135] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:25:17,202] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:25:17,270] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:25:17,386] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:25:17,498] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:25:17,607] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:25:17,674] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:25:17,739] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:25:17,803] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:25:17,867] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:25:17,931] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:25:18,038] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:25:18,044] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:25:18,046] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:25:18,048] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:25:18,050] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:25:18,052] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:25:18,053] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:25:18,054] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:25:18,055] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:25:18,057] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:25:18,059] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:25:18,061] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:25:18,062] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:25:18,064] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:25:18,066] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:25:18,068] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:25:19,478] INFO : VALIDATION | Step=27000/500000 | Train Loss=0.6375 | Valid Loss=11.2441 | Train Loss Unweighted=0.0663 | Valid Loss Unweighted=2.2967 | Stale=19 | ETA=935.3m +rank0 [2026-11-15 04:25:19,478] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:25:19,553] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:25:19,556] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:25:19,605] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:25:20,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:25:20,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:25:20,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:25:20,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:25:21,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:25:21,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:25:21,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:25:21,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:25:22,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:25:22,245] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:25:22,482] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:25:22,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:25:22,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:25:23,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:25:23,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:25:23,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:25:23,699] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 04:25:23,707] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:25:24,244] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_27000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:25:24,650] INFO : Plotted number of elements +rank0 [2026-11-15 04:25:25,819] INFO : Plotted jets +rank0 [2026-11-15 04:25:27,187] INFO : Plotted jet ratio +rank0 [2026-11-15 04:26:25,519] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 04:26:25,519] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:27:15,359] INFO : Step 28000/500000 rank0 | Train Loss: 0.4505 | Train Loss Unweighted: 0.0397 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0136 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0127 | Regression_eta: 0.0002 | Regression_pt: 0.0130 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0000 | Classification_binary: 0.0136 | Regression_cos_phi: 0.0270 | Regression_energy: 0.1582 | Regression_eta: 0.0274 | Regression_pt: 0.1668 | Regression_sin_phi: 0.0575 | LR: 6.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4831s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0029 | max_backward: 0.0392 | max_optimizer_step: 0.0640 | max_tensorboard_logging: 0.1282 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3401.2 | uss_MB: 2174.3 | child_rss_MB: 22736.9 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 04:27:15,361] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 04:27:15,428] INFO : Running validation on rank0 +rank0 [2026-11-15 04:27:15,428] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:27:15,428] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:27:15,546] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 04:27:15,653] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:27:15,753] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:27:15,822] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:27:15,886] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:27:15,950] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:27:16,016] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:27:16,083] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:27:16,195] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:27:16,308] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:27:16,422] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:27:16,490] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:27:16,557] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:27:16,622] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:27:16,688] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:27:16,755] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:27:16,862] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:27:16,868] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:27:16,870] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:27:16,872] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:27:16,874] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:27:16,876] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:27:16,877] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:27:16,879] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:27:16,881] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:27:16,882] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:27:16,883] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:27:16,885] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:27:16,886] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:27:16,888] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:27:16,890] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:27:16,892] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:27:18,305] INFO : VALIDATION | Step=28000/500000 | Train Loss=0.4505 | Valid Loss=11.3729 | Train Loss Unweighted=0.0397 | Valid Loss Unweighted=2.4492 | Stale=20 | ETA=933.4m +rank0 [2026-11-15 04:27:18,305] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:27:18,381] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:27:18,389] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:27:18,437] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:27:18,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:27:19,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:27:19,393] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:27:19,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:27:19,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:27:20,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:27:20,352] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:27:20,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:27:20,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:27:21,064] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:27:21,307] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:27:21,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:27:21,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:27:22,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:27:22,260] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:27:22,474] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:27:22,541] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:27:22,547] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:27:23,077] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_28000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:27:23,499] INFO : Plotted number of elements +rank0 [2026-11-15 04:27:24,652] INFO : Plotted jets +rank0 [2026-11-15 04:27:26,050] INFO : Plotted jet ratio +rank0 [2026-11-15 04:27:37,171] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 04:27:37,171] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:28:37,537] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 04:28:37,537] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:29:14,294] INFO : Step 29000/500000 rank0 | Train Loss: 0.2890 | Train Loss Unweighted: 0.0344 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0188 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0079 | Regression_eta: 0.0002 | Regression_pt: 0.0074 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0004 | Classification_binary: 0.0188 | Regression_cos_phi: 0.0216 | Regression_energy: 0.0990 | Regression_eta: 0.0232 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0319 | LR: 6.29e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4800s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0388 | max_optimizer_step: 0.0617 | max_tensorboard_logging: 0.1268 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3430.9 | uss_MB: 2204.0 | child_rss_MB: 22740.8 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:29:14,296] INFO : Running validation on rank0 +rank0 [2026-11-15 04:29:14,296] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:29:14,296] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:29:14,416] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 04:29:14,519] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:29:14,620] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:29:14,688] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:29:14,755] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:29:14,821] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:29:14,889] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:29:14,956] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:29:15,069] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:29:15,176] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:29:15,283] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:29:15,350] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:29:15,413] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:29:15,476] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:29:15,540] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:29:15,604] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:29:15,709] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:29:15,716] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:29:15,718] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:29:15,719] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:29:15,722] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:29:15,723] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:29:15,724] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:29:15,725] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:29:15,728] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:29:15,729] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:29:15,729] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:29:15,733] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:29:15,733] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:29:15,735] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:29:15,737] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:29:15,739] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:29:17,136] INFO : VALIDATION | Step=29000/500000 | Train Loss=0.2890 | Valid Loss=12.1393 | Train Loss Unweighted=0.0344 | Valid Loss Unweighted=2.6877 | Stale=21 | ETA=931.5m +rank0 [2026-11-15 04:29:17,136] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:29:17,210] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:29:17,214] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:29:17,276] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:29:17,765] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:29:18,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:29:18,246] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:29:18,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:29:18,721] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:29:18,955] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:29:19,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:29:19,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:29:19,681] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:29:19,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:29:20,173] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:29:20,408] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:29:20,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:29:20,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:29:21,113] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:29:21,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:29:21,389] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:29:21,398] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:29:21,929] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_29000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:29:22,336] INFO : Plotted number of elements +rank0 [2026-11-15 04:29:23,501] INFO : Plotted jets +rank0 [2026-11-15 04:29:24,871] INFO : Plotted jet ratio +rank0 [2026-11-15 04:29:49,488] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 04:29:49,488] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:30:50,407] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 04:30:50,407] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:31:13,956] INFO : Step 30000/500000 rank0 | Train Loss: 0.4761 | Train Loss Unweighted: 0.0908 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0729 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0082 | Regression_eta: 0.0006 | Regression_pt: 0.0088 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0000 | Classification_binary: 0.0729 | Regression_cos_phi: 0.0481 | Regression_energy: 0.1026 | Regression_eta: 0.0942 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0459 | LR: 6.58e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4955s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0354 | max_loss: 0.0026 | max_backward: 0.0388 | max_optimizer_step: 0.0633 | max_tensorboard_logging: 0.1357 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3454.5 | uss_MB: 2227.6 | child_rss_MB: 22740.0 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:31:13,957] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 04:31:14,027] INFO : Running validation on rank0 +rank0 [2026-11-15 04:31:14,027] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:31:14,027] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:31:14,152] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 04:31:14,256] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:31:14,355] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:31:14,423] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:31:14,488] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:31:14,551] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:31:14,617] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:31:14,682] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:31:14,795] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:31:14,905] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:31:15,013] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:31:15,080] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:31:15,145] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:31:15,210] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:31:15,276] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:31:15,341] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:31:15,449] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:31:15,456] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:31:15,458] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:31:15,459] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:31:15,460] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:31:15,464] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:31:15,464] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:31:15,466] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:31:15,468] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:31:15,469] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:31:15,471] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:31:15,473] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:31:15,473] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:31:15,475] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:31:15,477] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:31:15,479] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:31:16,891] INFO : VALIDATION | Step=30000/500000 | Train Loss=0.4761 | Valid Loss=11.8482 | Train Loss Unweighted=0.0908 | Valid Loss Unweighted=2.4545 | Stale=22 | ETA=929.8m +rank0 [2026-11-15 04:31:16,892] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:31:16,966] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:31:16,973] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:31:17,022] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:31:17,492] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:31:17,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:31:17,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:31:18,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:31:18,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:31:18,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:31:18,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:31:19,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:31:19,420] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:31:19,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:31:19,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:31:20,140] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:31:20,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:31:20,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:31:20,870] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:31:21,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:31:21,148] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:31:21,153] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:31:21,684] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:31:22,106] INFO : Plotted number of elements +rank0 [2026-11-15 04:31:23,289] INFO : Plotted jets +rank0 [2026-11-15 04:31:24,688] INFO : Plotted jet ratio +rank0 [2026-11-15 04:32:02,703] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 04:32:02,703] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:33:03,613] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 04:33:03,613] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:33:13,822] INFO : Step 31000/500000 rank0 | Train Loss: 0.6801 | Train Loss Unweighted: 0.0936 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0710 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0107 | Regression_eta: 0.0003 | Regression_pt: 0.0109 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0004 | Classification_binary: 0.0710 | Regression_cos_phi: 0.0883 | Regression_energy: 0.1332 | Regression_eta: 0.0419 | Regression_pt: 0.1399 | Regression_sin_phi: 0.2054 | LR: 6.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5201s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0359 | max_loss: 0.0026 | max_backward: 0.0385 | max_optimizer_step: 0.0778 | max_tensorboard_logging: 0.1337 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3402.5 | uss_MB: 2175.6 | child_rss_MB: 22739.8 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:33:13,823] INFO : Running validation on rank0 +rank0 [2026-11-15 04:33:13,823] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:33:13,824] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:33:13,943] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 04:33:14,055] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:33:14,159] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:33:14,226] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:33:14,291] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:33:14,355] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:33:14,420] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:33:14,486] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:33:14,597] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:33:14,698] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:33:14,796] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:33:14,864] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:33:14,929] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:33:14,994] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:33:15,061] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:33:15,126] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:33:15,234] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:33:15,241] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:33:15,243] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:33:15,245] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:33:15,247] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:33:15,249] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:33:15,250] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:33:15,252] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:33:15,254] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:33:15,254] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:33:15,256] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:33:15,258] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:33:15,259] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:33:15,261] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:33:15,263] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:33:15,265] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:33:16,666] INFO : VALIDATION | Step=31000/500000 | Train Loss=0.6801 | Valid Loss=11.9863 | Train Loss Unweighted=0.0936 | Valid Loss Unweighted=2.6181 | Stale=23 | ETA=928.1m +rank0 [2026-11-15 04:33:16,666] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:33:16,740] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:33:16,746] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:33:16,794] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:33:17,260] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:33:17,496] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:33:17,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:33:17,979] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:33:18,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:33:18,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:33:18,700] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:33:18,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:33:19,169] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:33:19,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:33:19,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:33:19,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:33:20,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:33:20,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:33:20,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:33:20,819] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:33:20,883] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-11-15 04:33:20,889] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:33:21,418] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_31000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:33:21,836] INFO : Plotted number of elements +rank0 [2026-11-15 04:33:22,997] INFO : Plotted jets +rank0 [2026-11-15 04:33:24,372] INFO : Plotted jet ratio +rank0 [2026-11-15 04:34:15,790] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-15 04:34:15,791] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:35:13,453] INFO : Step 32000/500000 rank0 | Train Loss: 1.9631 | Train Loss Unweighted: 0.1958 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1412 | Regression_cos_phi: 0.0002 | Regression_energy: 0.0255 | Regression_eta: 0.0070 | Regression_pt: 0.0217 | Regression_sin_phi: 0.0002 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0040 | Classification_binary: 0.1412 | Regression_cos_phi: 0.0691 | Regression_energy: 0.3176 | Regression_eta: 1.0349 | Regression_pt: 0.2777 | Regression_sin_phi: 0.1187 | LR: 7.19e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4949s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0361 | max_loss: 0.0026 | max_backward: 0.0392 | max_optimizer_step: 0.0652 | max_tensorboard_logging: 0.1284 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3511.6 | uss_MB: 2284.7 | child_rss_MB: 22738.0 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 04:35:13,455] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-32000.pth +rank0 [2026-11-15 04:35:13,525] INFO : Running validation on rank0 +rank0 [2026-11-15 04:35:13,525] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:35:13,525] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:35:13,646] INFO : saved batch0_epoch32000.parquet +rank0 [2026-11-15 04:35:13,753] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:35:13,859] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:35:13,927] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:35:13,992] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:35:14,056] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:35:14,122] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:35:14,188] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:35:14,301] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:35:14,411] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:35:14,520] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:35:14,587] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:35:14,653] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:35:14,718] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:35:14,784] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:35:14,850] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:35:14,963] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:35:14,969] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:35:14,972] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:35:14,974] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:35:14,976] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:35:14,977] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:35:14,979] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:35:14,979] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:35:14,982] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:35:14,983] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:35:14,985] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:35:14,986] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:35:14,988] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:35:14,990] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:35:14,992] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:35:14,994] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:35:16,424] INFO : VALIDATION | Step=32000/500000 | Train Loss=1.9631 | Valid Loss=12.7518 | Train Loss Unweighted=0.1958 | Valid Loss Unweighted=2.6648 | Stale=24 | ETA=926.4m +rank0 [2026-11-15 04:35:16,425] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:35:16,500] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:35:16,506] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:35:16,557] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:35:17,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:35:17,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:35:17,544] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:35:17,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:35:18,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:35:18,277] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:35:18,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:35:18,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:35:18,993] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:35:19,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:35:19,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:35:19,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:35:19,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:35:20,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:35:20,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:35:20,654] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:35:20,711] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:35:20,717] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:35:21,246] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_32000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:35:21,668] INFO : Plotted number of elements +rank0 [2026-11-15 04:35:22,910] INFO : Plotted jets +rank0 [2026-11-15 04:35:24,299] INFO : Plotted jet ratio +rank0 [2026-11-15 04:35:28,342] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-15 04:35:28,342] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:36:29,114] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-15 04:36:29,114] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:37:13,101] INFO : Step 33000/500000 rank0 | Train Loss: 1.2223 | Train Loss Unweighted: 0.2595 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1883 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0355 | Regression_eta: 0.0003 | Regression_pt: 0.0352 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.1883 | Regression_cos_phi: 0.0244 | Regression_energy: 0.4429 | Regression_eta: 0.0450 | Regression_pt: 0.4504 | Regression_sin_phi: 0.0712 | LR: 7.51e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5051s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0642 | max_tensorboard_logging: 0.1384 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3549.3 | uss_MB: 2322.4 | child_rss_MB: 22733.8 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:37:13,102] INFO : Running validation on rank0 +rank0 [2026-11-15 04:37:13,102] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:37:13,102] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:37:13,226] INFO : saved batch0_epoch33000.parquet +rank0 [2026-11-15 04:37:13,331] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:37:13,434] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:37:13,501] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:37:13,566] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:37:13,631] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:37:13,697] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:37:13,763] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:37:13,877] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:37:13,987] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:37:14,099] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:37:14,167] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:37:14,232] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:37:14,296] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:37:14,361] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:37:14,427] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:37:14,537] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:37:14,544] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:37:14,546] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:37:14,548] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:37:14,550] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:37:14,552] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:37:14,554] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:37:14,556] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:37:14,558] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:37:14,558] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:37:14,560] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:37:14,562] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:37:14,563] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:37:14,564] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:37:14,567] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:37:14,569] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:37:15,986] INFO : VALIDATION | Step=33000/500000 | Train Loss=1.2223 | Valid Loss=12.0923 | Train Loss Unweighted=0.2595 | Valid Loss Unweighted=2.6491 | Stale=25 | ETA=924.6m +rank0 [2026-11-15 04:37:15,986] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:37:16,059] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:37:16,065] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:37:16,115] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:37:16,586] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:37:16,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:37:17,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:37:17,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:37:17,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:37:17,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:37:18,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:37:18,274] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:37:18,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:37:18,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:37:18,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:37:19,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:37:19,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:37:19,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:37:19,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:37:20,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:37:20,230] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:37:20,235] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:37:20,758] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_33000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:37:21,176] INFO : Plotted number of elements +rank0 [2026-11-15 04:37:22,396] INFO : Plotted jets +rank0 [2026-11-15 04:37:23,749] INFO : Plotted jet ratio +rank0 [2026-11-15 04:37:41,036] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-15 04:37:41,037] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:38:41,372] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-15 04:38:41,372] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:39:12,038] INFO : Step 34000/500000 rank0 | Train Loss: 0.3936 | Train Loss Unweighted: 0.0395 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0204 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0091 | Regression_eta: 0.0003 | Regression_pt: 0.0095 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0009 | Classification_binary: 0.0204 | Regression_cos_phi: 0.0260 | Regression_energy: 0.1131 | Regression_eta: 0.0506 | Regression_pt: 0.1218 | Regression_sin_phi: 0.0609 | LR: 7.83e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4915s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0363 | max_loss: 0.0026 | max_backward: 0.0385 | max_optimizer_step: 0.0624 | max_tensorboard_logging: 0.1323 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3497.2 | uss_MB: 2270.3 | child_rss_MB: 22740.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 04:39:12,040] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-34000.pth +rank0 [2026-11-15 04:39:12,106] INFO : Running validation on rank0 +rank0 [2026-11-15 04:39:12,107] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:39:12,107] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:39:12,225] INFO : saved batch0_epoch34000.parquet +rank0 [2026-11-15 04:39:12,332] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:39:12,432] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:39:12,501] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:39:12,567] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:39:12,633] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:39:12,700] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:39:12,768] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:39:12,882] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:39:12,993] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:39:13,104] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:39:13,171] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:39:13,236] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:39:13,299] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:39:13,363] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:39:13,428] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:39:13,536] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:39:13,542] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:39:13,544] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:39:13,547] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:39:13,549] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:39:13,550] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:39:13,551] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:39:13,553] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:39:13,555] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:39:13,556] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:39:13,556] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:39:13,559] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:39:13,560] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:39:13,561] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:39:13,564] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:39:13,566] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:39:14,974] INFO : VALIDATION | Step=34000/500000 | Train Loss=0.3936 | Valid Loss=12.1877 | Train Loss Unweighted=0.0395 | Valid Loss Unweighted=2.8850 | Stale=26 | ETA=922.6m +rank0 [2026-11-15 04:39:14,975] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:39:15,049] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:39:15,056] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:39:15,107] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:39:15,594] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:39:15,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:39:16,084] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:39:16,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:39:16,563] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:39:16,796] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:39:17,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:39:17,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:39:17,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:39:17,756] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:39:17,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:39:18,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:39:18,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:39:18,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:39:18,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:39:19,156] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:39:19,219] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:39:19,226] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:39:19,774] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_34000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:39:20,185] INFO : Plotted number of elements +rank0 [2026-11-15 04:39:21,353] INFO : Plotted jets +rank0 [2026-11-15 04:39:22,713] INFO : Plotted jet ratio +rank0 [2026-11-15 04:39:53,367] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-15 04:39:53,368] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:40:53,442] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-15 04:40:53,442] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:10,620] INFO : Step 35000/500000 rank0 | Train Loss: 0.8852 | Train Loss Unweighted: 0.0856 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.0620 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0109 | Regression_eta: 0.0003 | Regression_pt: 0.0110 | Regression_sin_phi: 0.0004 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0110 | Classification_binary: 0.0620 | Regression_cos_phi: 0.2261 | Regression_energy: 0.1363 | Regression_eta: 0.0515 | Regression_pt: 0.1409 | Regression_sin_phi: 0.2574 | LR: 8.16e-05 | DataLoad Time: 0.0059s | Model Forward Time: 0.5108s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0389 | max_loss: 0.0031 | max_backward: 0.0415 | max_optimizer_step: 0.0693 | max_tensorboard_logging: 0.1289 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3601.9 | uss_MB: 2375.6 | child_rss_MB: 22740.8 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 14.0 +rank0 [2026-11-15 04:41:10,621] INFO : Running validation on rank0 +rank0 [2026-11-15 04:41:10,621] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:41:10,621] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:41:10,740] INFO : saved batch0_epoch35000.parquet +rank0 [2026-11-15 04:41:10,844] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:41:10,946] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:41:11,015] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:41:11,079] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:41:11,142] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:41:11,205] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:41:11,269] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:41:11,381] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:41:11,493] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:41:11,602] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:41:11,669] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:41:11,735] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:41:11,801] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:41:11,866] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:41:11,933] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:41:12,040] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:41:12,047] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:41:12,049] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:41:12,051] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:41:12,051] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:41:12,055] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:41:12,055] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:41:12,057] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:41:12,059] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:41:12,060] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:41:12,062] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:41:12,063] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:41:12,064] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:41:12,066] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:41:12,067] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:41:12,070] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:41:13,449] INFO : VALIDATION | Step=35000/500000 | Train Loss=0.8852 | Valid Loss=12.7311 | Train Loss Unweighted=0.0856 | Valid Loss Unweighted=2.9098 | Stale=27 | ETA=920.6m +rank0 [2026-11-15 04:41:13,449] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:41:13,522] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:41:13,528] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:41:13,577] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:41:14,049] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:41:14,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:41:14,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:41:14,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:41:15,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:41:15,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:41:15,500] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:41:15,737] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:41:15,971] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:41:16,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:41:16,453] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:41:16,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:41:16,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:41:17,176] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:41:17,411] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:41:17,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:41:17,686] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:41:17,691] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:41:18,221] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_35000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:41:18,642] INFO : Plotted number of elements +rank0 [2026-11-15 04:41:19,803] INFO : Plotted jets +rank0 [2026-11-15 04:41:21,188] INFO : Plotted jet ratio +rank0 [2026-11-15 04:42:05,242] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-15 04:42:05,242] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:43:05,531] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-15 04:43:05,532] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:43:09,353] INFO : Step 36000/500000 rank0 | Train Loss: 0.4882 | Train Loss Unweighted: 0.1337 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.1173 | Regression_cos_phi: 0.0004 | Regression_energy: 0.0080 | Regression_eta: 0.0001 | Regression_pt: 0.0079 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0000 | Classification_binary: 0.1173 | Regression_cos_phi: 0.1240 | Regression_energy: 0.1003 | Regression_eta: 0.0099 | Regression_pt: 0.1010 | Regression_sin_phi: 0.0358 | LR: 8.50e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4825s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0631 | max_tensorboard_logging: 0.1296 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3626.7 | uss_MB: 2439.8 | child_rss_MB: 22740.0 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:43:09,355] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-36000.pth +rank0 [2026-11-15 04:43:09,423] INFO : Running validation on rank0 +rank0 [2026-11-15 04:43:09,423] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:43:09,423] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:43:09,542] INFO : saved batch0_epoch36000.parquet +rank0 [2026-11-15 04:43:09,647] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:43:09,747] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:43:09,816] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:43:09,882] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:43:09,947] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:43:10,014] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:43:10,082] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:43:10,196] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:43:10,300] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:43:10,404] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:43:10,471] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:43:10,538] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:43:10,604] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:43:10,669] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:43:10,735] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:43:10,843] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:43:10,850] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:43:10,852] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:43:10,854] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:43:10,856] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:43:10,858] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:43:10,858] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:43:10,859] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:43:10,862] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:43:10,863] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:43:10,865] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:43:10,865] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:43:10,867] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:43:10,869] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:43:10,870] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:43:10,873] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:43:12,270] INFO : VALIDATION | Step=36000/500000 | Train Loss=0.4882 | Valid Loss=12.0784 | Train Loss Unweighted=0.1337 | Valid Loss Unweighted=2.8195 | Stale=28 | ETA=918.6m +rank0 [2026-11-15 04:43:12,270] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:43:12,347] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:43:12,355] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:43:12,407] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:43:12,906] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:43:13,154] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:43:13,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:43:13,635] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:43:13,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:43:14,113] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:43:14,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:43:14,582] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:43:14,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:43:15,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:43:15,321] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:43:15,562] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:43:15,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:43:16,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:43:16,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:43:16,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:43:16,546] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:43:16,551] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:43:17,090] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_36000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:43:17,502] INFO : Plotted number of elements +rank0 [2026-11-15 04:43:18,658] INFO : Plotted jets +rank0 [2026-11-15 04:43:20,045] INFO : Plotted jet ratio +rank0 [2026-11-15 04:44:17,446] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-15 04:44:17,446] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:45:08,538] INFO : Step 37000/500000 rank0 | Train Loss: 0.3716 | Train Loss Unweighted: 0.0653 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0444 | Regression_cos_phi: 0.0001 | Regression_energy: 0.0103 | Regression_eta: 0.0001 | Regression_pt: 0.0103 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0001 | Classification_binary: 0.0444 | Regression_cos_phi: 0.0172 | Regression_energy: 0.1282 | Regression_eta: 0.0203 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0296 | LR: 8.85e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4915s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0356 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0636 | max_tensorboard_logging: 0.1301 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3644.7 | uss_MB: 2472.3 | child_rss_MB: 22736.2 | allocated_MB: 170.0 | reserved_MB: 294.0 | inactive_split_MB: 18.0 +rank0 [2026-11-15 04:45:08,540] INFO : Running validation on rank0 +rank0 [2026-11-15 04:45:08,540] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:45:08,540] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:45:08,660] INFO : saved batch0_epoch37000.parquet +rank0 [2026-11-15 04:45:08,766] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:45:08,868] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:45:08,936] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:45:09,003] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:45:09,070] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:45:09,137] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:45:09,205] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:45:09,301] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:45:09,401] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:45:09,502] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:45:09,568] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:45:09,633] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:45:09,696] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:45:09,760] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:45:09,824] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:45:09,931] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:45:09,937] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:45:09,939] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:45:09,941] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:45:09,943] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:45:09,945] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:45:09,946] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:45:09,949] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:45:09,951] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:45:09,952] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:45:09,954] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:45:09,956] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:45:09,956] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:45:09,958] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:45:09,960] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:45:09,962] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:45:11,359] INFO : VALIDATION | Step=37000/500000 | Train Loss=0.3716 | Valid Loss=11.7303 | Train Loss Unweighted=0.0653 | Valid Loss Unweighted=2.7800 | Stale=29 | ETA=916.7m +rank0 [2026-11-15 04:45:11,359] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:45:11,435] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:45:11,442] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:45:11,493] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:45:11,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:45:12,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:45:12,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:45:12,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:45:12,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:45:13,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:45:13,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:45:13,675] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:45:13,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:45:14,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:45:14,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:45:14,650] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:45:14,887] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:45:15,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:45:15,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:45:15,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:45:15,636] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:45:15,641] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:45:16,182] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_37000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:45:16,592] INFO : Plotted number of elements +rank0 [2026-11-15 04:45:17,756] INFO : Plotted jets +rank0 [2026-11-15 04:45:19,128] INFO : Plotted jet ratio +rank0 [2026-11-15 04:45:29,407] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-15 04:45:29,407] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:46:30,183] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-15 04:46:30,183] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:47:07,809] INFO : Step 38000/500000 rank0 | Train Loss: 0.8592 | Train Loss Unweighted: 0.0642 | Train Loss Components: Classification: 0.0000 | Classification_binary: 0.0116 | Regression_cos_phi: 0.0003 | Regression_energy: 0.0265 | Regression_eta: 0.0002 | Regression_pt: 0.0254 | Regression_sin_phi: 0.0001 | Task Weights: Classification: 92.0252 | Classification_binary: 1.0000 | Regression_cos_phi: 296.8046 | Regression_energy: 12.4653 | Regression_eta: 147.8529 | Regression_pt: 12.8030 | Regression_sin_phi: 582.6146 | Task Weighted Losses: Classification: 0.0000 | Classification_binary: 0.0116 | Regression_cos_phi: 0.0945 | Regression_energy: 0.3309 | Regression_eta: 0.0298 | Regression_pt: 0.3253 | Regression_sin_phi: 0.0671 | LR: 9.21e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4887s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0385 | max_optimizer_step: 0.0633 | max_tensorboard_logging: 0.1308 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3677.4 | uss_MB: 2505.0 | child_rss_MB: 22739.4 | allocated_MB: 170.0 | reserved_MB: 292.0 | inactive_split_MB: 16.0 +rank0 [2026-11-15 04:47:07,811] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/checkpoints/checkpoint-38000.pth +rank0 [2026-11-15 04:47:07,878] INFO : Running validation on rank0 +rank0 [2026-11-15 04:47:07,878] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:47:07,878] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:47:08,007] INFO : saved batch0_epoch38000.parquet +rank0 [2026-11-15 04:47:08,112] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:47:08,214] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:47:08,283] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:47:08,350] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:47:08,417] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:47:08,485] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:47:08,552] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:47:08,666] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:47:08,774] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:47:08,882] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:47:08,948] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:47:09,012] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:47:09,076] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:47:09,142] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:47:09,206] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:47:09,319] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:47:09,325] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:47:09,326] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:47:09,329] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:47:09,331] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:47:09,333] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:47:09,335] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:47:09,337] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:47:09,339] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:47:09,340] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:47:09,342] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:47:09,344] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:47:09,344] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:47:09,346] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:47:09,348] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:47:09,350] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:47:10,753] INFO : VALIDATION | Step=38000/500000 | Train Loss=0.8592 | Valid Loss=12.0072 | Train Loss Unweighted=0.0642 | Valid Loss Unweighted=3.0325 | Stale=30 | ETA=914.8m +rank0 [2026-11-15 04:47:10,753] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:47:10,828] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-15 04:47:10,834] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-15 04:47:10,884] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:47:11,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-15 04:47:11,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-15 04:47:11,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-15 04:47:12,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-15 04:47:12,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-15 04:47:12,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-15 04:47:12,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-15 04:47:13,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-15 04:47:13,337] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-15 04:47:13,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-15 04:47:13,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-15 04:47:14,056] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-15 04:47:14,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-15 04:47:14,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-15 04:47:14,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-15 04:47:14,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-15 04:47:15,045] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:47:15,051] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:47:15,585] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_ecal_4mm_seed12345_20261115_033130_320062/preds_step_38000/cld_edm_z_tautau_pf +rank0 [2026-11-15 04:47:15,990] INFO : Plotted number of elements +rank0 [2026-11-15 04:47:17,238] INFO : Plotted jets +rank0 [2026-11-15 04:47:18,607] INFO : Plotted jet ratio +rank0 [2026-11-15 04:47:18,944] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 04:47:18,944] INFO : Training completed. Total time on device 0: 75.383min diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..ab2a3df07099e03462fc0c256d848b5ebeb65a3e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcac522406327f951dfecfa3a5902aeb7198340c22bf3d7f947fb08f55b51e59 +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/checkpoints/checkpoint-34000.pth b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/checkpoints/checkpoint-34000.pth new file mode 100644 index 0000000000000000000000000000000000000000..b8aae95ab9467225f9a77d26f6f032e4a4a0ad99 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/checkpoints/checkpoint-34000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9800a5b3c88b1d57fbfa5b3a3d1aefc41ea8e246822c0ea433e7be9806a5f644 +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/history/step_1000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/history/step_1000.json new file mode 100644 index 0000000000000000000000000000000000000000..4a13b7bc2e8d8b212cc4a318329ddc6f4a590c65 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/history/step_1000.json @@ -0,0 +1 @@ +{"epoch": 1.7793594306049823, "train": {"Classification_binary": 0.9526971578598022, "Classification": 0.005531268659979105, "Regression_pt": 0.016782797873020172, "Regression_eta": 0.00028991466388106346, "Regression_sin_phi": 0.00025613713660277426, "Regression_cos_phi": 0.0020202945452183485, "Regression_energy": 0.01625811867415905, "Total": 2.729661464691162, "Total_unweighted": 0.9938356894126628}, "train_group": {}, "valid": {"Classification_binary": 1.0317163467407227, "Classification": 0.00625403318554163, "Regression_pt": 0.06455358117818832, "Regression_eta": 0.02343999780714512, "Regression_sin_phi": 0.0010068592382594943, "Regression_cos_phi": 0.00047689685015939176, "Regression_energy": 0.07567848265171051, "Total": 7.202181339263916, "Total_unweighted": 1.2031261976517271, "metrics/particle/class_1/efficiency": 0.960824728012085, "metrics/particle/class_1/pid_accuracy": 0.9849785566329956, "metrics/particle/class_2/efficiency": 0.7727272510528564, "metrics/particle/class_2/pid_accuracy": 0.8823529481887817, "metrics/particle/class_3/efficiency": 0.8842504620552063, "metrics/particle/class_3/pid_accuracy": 0.9978540539741516, "metrics/particle/class_4/efficiency": 0.9583333134651184, "metrics/particle/class_4/pid_accuracy": 0.9239130616188049, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9878048896789551, "metrics/particle/count/bias_mean": -0.03999999910593033, "metrics/particle/count/mae": 0.24799999594688416, "metrics/particle/count/prediction_mean": 4.808000087738037, "metrics/particle/count/target_mean": 4.8480000495910645, "metrics/particle/event/energy_relative_abs_error": 0.037531327456235886, "metrics/particle/event/energy_response_mean": 0.9922536611557007, "metrics/particle/event/met_abs_error": 1.0819724798202515, "metrics/particle/event/scalar_pt_relative_abs_error": 0.035317908972501755, "metrics/particle/event/scalar_pt_response_mean": 0.9939746856689453, "metrics/particle/event/vector_pt_closure": 0.040704868733882904, "metrics/particle/matched/delta_eta_abs_mean": 0.008561627008020878, "metrics/particle/matched/delta_phi_abs_mean": 0.007300459314137697, "metrics/particle/matched/delta_r_mean": 0.01253063976764679, "metrics/particle/matched/energy_relative_abs_mean": 0.06243151053786278, "metrics/particle/matched/pid_accuracy": 0.9839714765548706, "metrics/particle/matched/pt_relative_abs_mean": 0.06264891475439072, "metrics/particle/matching/duplicate_fraction": 0.019134774804115295, "metrics/particle/matching/efficiency": 0.9265676736831665, "metrics/particle/matching/f1": 0.9304059743881226, "metrics/particle/matching/purity": 0.9342762231826782, "metrics/particle/matching/target_coverage_dr0p05": 0.9100660085678101, "metrics/particle/matching/target_coverage_dr0p10": 0.9471946954727173, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8564356565475464, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9389439225196838}, "step_train_time": 0.48804521560668945, "step_valid_time": 2.5873908739995386, "step_total_time": 110.94677472114563, "elapsed_training_loop_seconds": 110.94677472114563, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0018339126975837, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.023876591399858427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 433, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 432, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9953917050691244, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9976905311778291, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9965397923875432, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0023094688221709007, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9930875576036866, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9953810623556582, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.994232987312572, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9953917050691244, "step_test_time": 4.437857344999429, "step_plot_time": 3.9134936509999534, "step_validation_cycle_time": 11.312120313000378} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..ddd1b4265e8b63f88fd0c46ebaa1856ad6cb560c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 0.7509037256240845, "Classification": 0.00012693647295236588, "Regression_pt": 0.02356974221765995, "Regression_eta": 0.0008063416462391615, "Regression_sin_phi": 9.649820276536047e-05, "Regression_cos_phi": 0.00013338006101548672, "Regression_energy": 0.02156198024749756, "Total": 1.5153295993804932, "Total_unweighted": 0.7971986044722144}, "train_group": {}, "valid": {"Classification_binary": 1.052809238433838, "Classification": 0.006705285515636206, "Regression_pt": 0.05421711131930351, "Regression_eta": 0.023223405703902245, "Regression_sin_phi": 0.001073984196409583, "Regression_cos_phi": 0.0003817574179265648, "Regression_energy": 0.06872911751270294, "Total": 7.008184432983398, "Total_unweighted": 1.207139900099719, "metrics/particle/class_1/efficiency": 0.9670103192329407, "metrics/particle/class_1/pid_accuracy": 0.9893389940261841, "metrics/particle/class_2/efficiency": 0.7272727489471436, "metrics/particle/class_2/pid_accuracy": 0.875, "metrics/particle/class_3/efficiency": 0.9032257795333862, "metrics/particle/class_3/pid_accuracy": 0.9957982897758484, "metrics/particle/class_4/efficiency": 0.9895833134651184, "metrics/particle/class_4/pid_accuracy": 0.9578947424888611, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.024000000208616257, "metrics/particle/count/mae": 0.23999999463558197, "metrics/particle/count/prediction_mean": 4.823999881744385, "metrics/particle/count/target_mean": 4.8480000495910645, "metrics/particle/event/energy_relative_abs_error": 0.03569292277097702, "metrics/particle/event/energy_response_mean": 0.9852526783943176, "metrics/particle/event/met_abs_error": 1.0731918811798096, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03298186883330345, "metrics/particle/event/scalar_pt_response_mean": 0.9878531694412231, "metrics/particle/event/vector_pt_closure": 0.03752328082919121, "metrics/particle/matched/delta_eta_abs_mean": 0.006944244261831045, "metrics/particle/matched/delta_phi_abs_mean": 0.006445564795285463, "metrics/particle/matched/delta_r_mean": 0.010583597235381603, "metrics/particle/matched/energy_relative_abs_mean": 0.06087876856327057, "metrics/particle/matched/pid_accuracy": 0.9885764718055725, "metrics/particle/matched/pt_relative_abs_mean": 0.061132896691560745, "metrics/particle/matching/duplicate_fraction": 0.016583748161792755, "metrics/particle/matching/efficiency": 0.9389439225196838, "metrics/particle/matching/f1": 0.9412738084793091, "metrics/particle/matching/purity": 0.9436152577400208, "metrics/particle/matching/target_coverage_dr0p05": 0.9282178282737732, "metrics/particle/matching/target_coverage_dr0p10": 0.9537953734397888, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8696369528770447, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9488449096679688}, "step_train_time": 0.47765350341796875, "step_valid_time": 2.8117784239993853, "step_total_time": 1172.0111708641052, "elapsed_training_loop_seconds": 1172.0111708641052, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9998442830125798, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.023731025923810822, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907834101382489, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9976798143851509, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9942196531791907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.002320185614849188, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.988479262672811, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9953596287703016, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.991907514450867, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907834101382489, "step_test_time": 4.218191066000145, "step_plot_time": 3.485251782999512, "step_validation_cycle_time": 10.838728402000015} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/history/step_11000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/history/step_11000.json new file mode 100644 index 0000000000000000000000000000000000000000..3fcf929a707f50af255f2690ac1cd6a647b993ea --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/history/step_11000.json @@ -0,0 +1 @@ +{"epoch": 19.572953736654803, "train": {"Classification_binary": 0.584793210029602, "Classification": 0.00094695232110098, "Regression_pt": 0.01681862212717533, "Regression_eta": 0.0002808306599035859, "Regression_sin_phi": 0.00017327838577330112, "Regression_cos_phi": 8.185053593479097e-05, "Regression_energy": 0.01575174182653427, "Total": 1.1988170146942139, "Total_unweighted": 0.6188464858860243}, "train_group": {}, "valid": {"Classification_binary": 1.0752066373825073, "Classification": 0.006983731873333454, "Regression_pt": 0.05855454131960869, "Regression_eta": 0.023229975253343582, "Regression_sin_phi": 0.0011338925687596202, "Regression_cos_phi": 0.00043374786037020385, "Regression_energy": 0.07336965203285217, "Total": 7.206864356994629, "Total_unweighted": 1.238912178290775, "metrics/particle/class_1/efficiency": 0.9670103192329407, "metrics/particle/class_1/pid_accuracy": 0.9914712309837341, "metrics/particle/class_2/efficiency": 0.7727272510528564, "metrics/particle/class_2/pid_accuracy": 0.9411764740943909, "metrics/particle/class_3/efficiency": 0.8956356644630432, "metrics/particle/class_3/pid_accuracy": 0.9957627058029175, "metrics/particle/class_4/efficiency": 0.96875, "metrics/particle/class_4/pid_accuracy": 0.9677419066429138, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9878048896789551, "metrics/particle/count/bias_mean": -0.052000001072883606, "metrics/particle/count/mae": 0.23600000143051147, "metrics/particle/count/prediction_mean": 4.796000003814697, "metrics/particle/count/target_mean": 4.8480000495910645, "metrics/particle/event/energy_relative_abs_error": 0.03762602061033249, "metrics/particle/event/energy_response_mean": 0.9844349026679993, "metrics/particle/event/met_abs_error": 1.0564825534820557, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0349871926009655, "metrics/particle/event/scalar_pt_response_mean": 0.9866034984588623, "metrics/particle/event/vector_pt_closure": 0.03874443471431732, "metrics/particle/matched/delta_eta_abs_mean": 0.006963040679693222, "metrics/particle/matched/delta_phi_abs_mean": 0.006758529227226973, "metrics/particle/matched/delta_r_mean": 0.010877125896513462, "metrics/particle/matched/energy_relative_abs_mean": 0.06137945502996445, "metrics/particle/matched/pid_accuracy": 0.9902912378311157, "metrics/particle/matched/pt_relative_abs_mean": 0.06169331818819046, "metrics/particle/matching/duplicate_fraction": 0.01668056659400463, "metrics/particle/matching/efficiency": 0.9348185062408447, "metrics/particle/matching/f1": 0.9398589730262756, "metrics/particle/matching/purity": 0.9449541568756104, "metrics/particle/matching/target_coverage_dr0p05": 0.919966995716095, "metrics/particle/matching/target_coverage_dr0p10": 0.9513201117515564, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8638613820075989, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9455445408821106}, "step_train_time": 0.4832000732421875, "step_valid_time": 2.8182203149999623, "step_total_time": 1290.126191854477, "elapsed_training_loop_seconds": 1290.126191854477, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0003667328515544, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.02571458216721223, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9930875576036866, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 1.0, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9965317919075144, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9907834101382489, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9976798143851509, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9942196531791907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9930875576036866, "step_test_time": 4.301301040999533, "step_plot_time": 3.5185735530003512, "step_validation_cycle_time": 10.963594918000126} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/history/step_12000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/history/step_12000.json new file mode 100644 index 0000000000000000000000000000000000000000..03d1c6bf6eb6e7f93ba0721157244084323e6d77 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/history/step_12000.json @@ -0,0 +1 @@ +{"epoch": 21.352313167259787, "train": {"Classification_binary": 0.6775558590888977, "Classification": 0.00030206411611288786, "Regression_pt": 0.024349326267838478, "Regression_eta": 0.0001668758923187852, "Regression_sin_phi": 0.00010564008698565885, "Regression_cos_phi": 0.00030069632339291275, "Regression_energy": 0.0240508820861578, "Total": 1.471601128578186, "Total_unweighted": 0.7268313438617042}, "train_group": {}, "valid": {"Classification_binary": 1.1356700658798218, "Classification": 0.008777795359492302, "Regression_pt": 0.05911214277148247, "Regression_eta": 0.02338225767016411, "Regression_sin_phi": 0.001110913697630167, "Regression_cos_phi": 0.0004643246065825224, "Regression_energy": 0.07371162623167038, "Total": 7.454102993011475, "Total_unweighted": 1.3022291262168437, "metrics/particle/class_1/efficiency": 0.969072163105011, "metrics/particle/class_1/pid_accuracy": 0.9872340559959412, "metrics/particle/class_2/efficiency": 0.7272727489471436, "metrics/particle/class_2/pid_accuracy": 0.875, "metrics/particle/class_3/efficiency": 0.8975331783294678, "metrics/particle/class_3/pid_accuracy": 0.9957716464996338, "metrics/particle/class_4/efficiency": 0.9895833134651184, "metrics/particle/class_4/pid_accuracy": 0.9684210419654846, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9878048896789551, "metrics/particle/count/bias_mean": -0.012000000104308128, "metrics/particle/count/mae": 0.23600000143051147, "metrics/particle/count/prediction_mean": 4.835999965667725, "metrics/particle/count/target_mean": 4.8480000495910645, "metrics/particle/event/energy_relative_abs_error": 0.0379168726503849, "metrics/particle/event/energy_response_mean": 0.9828788042068481, "metrics/particle/event/met_abs_error": 1.0479824542999268, "metrics/particle/event/scalar_pt_relative_abs_error": 0.034914325922727585, "metrics/particle/event/scalar_pt_response_mean": 0.9861527681350708, "metrics/particle/event/vector_pt_closure": 0.0383915938436985, "metrics/particle/matched/delta_eta_abs_mean": 0.006982996594160795, "metrics/particle/matched/delta_phi_abs_mean": 0.006969411391764879, "metrics/particle/matched/delta_r_mean": 0.011080920696258545, "metrics/particle/matched/energy_relative_abs_mean": 0.061543747782707214, "metrics/particle/matched/pid_accuracy": 0.9876760840415955, "metrics/particle/matched/pt_relative_abs_mean": 0.06161850318312645, "metrics/particle/matching/duplicate_fraction": 0.017369726672768593, "metrics/particle/matching/efficiency": 0.9372937083244324, "metrics/particle/matching/f1": 0.9384551644325256, "metrics/particle/matching/purity": 0.9396195411682129, "metrics/particle/matching/target_coverage_dr0p05": 0.9257425665855408, "metrics/particle/matching/target_coverage_dr0p10": 0.9513201117515564, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8688119053840637, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9488449096679688}, "step_train_time": 0.49674057960510254, "step_valid_time": 2.7786229089997505, "step_total_time": 1409.6266541481018, "elapsed_training_loop_seconds": 1409.6266541481018, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9992366286867638, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.023418117805286576, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907834101382489, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 1.0, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9953703703703703, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.988479262672811, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9976744186046511, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9930555555555556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907834101382489, "step_test_time": 4.203700459000174, "step_plot_time": 3.5010038989994428, "step_validation_cycle_time": 10.801421057999505} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/history/step_13000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/history/step_13000.json new file mode 100644 index 0000000000000000000000000000000000000000..377831b201e1571e31f36997d254509c56b128b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_ecal_5p1mm_seed12345_20261115_033229_489141/history/step_13000.json @@ -0,0 +1 @@ +{"epoch": 23.131672597864767, "train": {"Classification_binary": 0.3838437795639038, "Classification": 0.0006091101095080376, "Regression_pt": 0.021144408732652664, "Regression_eta": 0.0002780222857836634, "Regression_sin_phi": 0.00014271856343839318, "Regression_cos_phi": 0.00011000780068570748, "Regression_energy": 0.019802702590823174, "Total": 1.0703320503234863, "Total_unweighted": 0.42593074964679545}, "train_group": {}, "valid": {"Classification_binary": 1.1995292901992798, "Classification": 0.008845595642924309, "Regression_pt": 0.05740884318947792, "Regression_eta": 0.02346317656338215, "Regression_sin_phi": 0.0011535113444551826, "Regression_cos_phi": 0.00047180880210362375, "Regression_energy": 0.07314906269311905, "Total": 7.52656364440918, "Total_unweighted": 1.364021288434742, "metrics/particle/class_1/efficiency": 0.9670103192329407, "metrics/particle/class_1/pid_accuracy": 0.9850746393203735, "metrics/particle/class_2/efficiency": 0.7272727489471436, "metrics/particle/class_2/pid_accuracy": 0.9375, "metrics/particle/class_3/efficiency": 0.8880455493927002, "metrics/particle/class_3/pid_accuracy": 0.9935897588729858, "metrics/particle/class_4/efficiency": 0.9791666865348816, "metrics/particle/class_4/pid_accuracy": 0.978723406791687, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.06400000303983688, "metrics/particle/count/mae": 0.2639999985694885, "metrics/particle/count/prediction_mean": 4.783999919891357, "metrics/particle/count/target_mean": 4.8480000495910645, "metrics/particle/event/energy_relative_abs_error": 0.03739628940820694, "metrics/particle/event/energy_response_mean": 0.9892486333847046, "metrics/particle/event/met_abs_error": 1.0875229835510254, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03463468328118324, "metrics/particle/event/scalar_pt_response_mean": 0.9909713268280029, "metrics/particle/event/vector_pt_closure": 0.03892432525753975, "metrics/particle/matched/delta_eta_abs_mean": 0.006844100076705217, "metrics/particle/matched/delta_phi_abs_mean": 0.00701893912628293, "metrics/particle/matched/delta_r_mean": 0.011012664996087551, "metrics/particle/matched/energy_relative_abs_mean": 0.0604417584836483, "metrics/particle/matched/pid_accuracy": 0.9884853959083557, "metrics/particle/matched/pt_relative_abs_mean": 0.06054140254855156, "metrics/particle/matching/duplicate_fraction": 0.016722407191991806, "metrics/particle/matching/efficiency": 0.9315181374549866, "metrics/particle/matching/f1": 0.9377076625823975, "metrics/particle/matching/purity": 0.9439799189567566, "metrics/particle/matching/target_coverage_dr0p05": 0.9191418886184692, "metrics/particle/matching/target_coverage_dr0p10": 0.9463696479797363, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.8638613820075989, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9414191246032715}, "step_train_time": 0.4803483486175537, "step_valid_time": 2.7743202359997667, "step_total_time": 1527.9847218990326, "elapsed_training_loop_seconds": 1527.9847218990326, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0006905049484818, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.024758588781563784, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9907834101382489, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 1.0, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9953703703703703, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.988479262672811, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9976744186046511, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9930555555555556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9907834101382489, "step_test_time": 4.233181200999752, "step_plot_time": 3.5207575679996808, "step_validation_cycle_time": 10.860316358999626} \ No newline at end of file