diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..d9df5d6a5ebe1b8d9fbaef5ce46226c2365752d5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,152 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 24 +0, 899 +1, 899 +2, 901 +3, 913 +4, 903 +5, 909 +6, 909 +7, 901 +0, 1955 +1, 1955 +2, 1955 +3, 1955 +4, 1955 +5, 1955 +6, 1955 +7, 1955 +0, 2411 +1, 2405 +2, 2413 +3, 2409 +4, 2417 +5, 2427 +6, 2417 +7, 2413 +0, 25239 +1, 25239 +2, 25239 +3, 25239 +4, 25239 +5, 25239 +6, 25239 +7, 25239 +0, 28863 +1, 28929 +2, 28773 +3, 28787 +4, 28781 +5, 28761 +6, 28915 +7, 28997 +0, 38975 +1, 39047 +2, 38905 +3, 38869 +4, 38897 +5, 38895 +6, 39009 +7, 39049 +0, 39773 +1, 39857 +2, 39707 +3, 39705 +4, 39705 +5, 39703 +6, 39839 +7, 39857 +0, 39773 +1, 39857 +2, 39707 +3, 39705 +4, 39705 +5, 39703 +6, 39839 +7, 39857 +0, 39783 +1, 39867 +2, 39717 +3, 39717 +4, 39717 +5, 39713 +6, 39849 +7, 39867 +0, 39783 +1, 39867 +2, 39717 +3, 39717 +4, 39717 +5, 39713 +6, 39849 +7, 39867 +0, 109499 +1, 109597 +2, 109441 +3, 109445 +4, 109451 +5, 109447 +6, 109573 +7, 109607 +0, 110101 +1, 110185 +2, 110035 +3, 110035 +4, 110035 +5, 110031 +6, 110167 +7, 110185 +0, 110105 +1, 110191 +2, 110041 +3, 110039 +4, 110039 +5, 110037 +6, 110171 +7, 110191 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/run.env b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..9e5b865f93757b40724859a62d0a2bd85e791822 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-better-memory-dump-support +commit=c3a8a8a7 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.00 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=768 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..320d2eb5b1ed2b31198eaa33bd83f132ed6a77c6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log @@ -0,0 +1,1483 @@ +============================================================== + DSV2_LITE_L24 / ultraep (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.00 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ c3a8a8a7 +============================================================== +W0826 19:20:25.163000 652586 torch/distributed/run.py:766] +W0826 19:20:25.163000 652586 torch/distributed/run.py:766] ***************************************** +W0826 19:20:25.163000 652586 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:20:25.163000 652586 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 8 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1[Gloo] Rank is connected to 73 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to [Gloo] Rank 74 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank1]:[W826 19:20:43.422866997 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:20:43.423261437 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:20:43.430152150 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:20:43.430154133 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:20:43.430195653 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:20:43.430219884 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:20:43.430239344 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:20:43.585531427 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.451 seconds +time to initialize megatron (seconds): 6.449 +[after megatron is initialized] datetime: 2026-08-26 19:20:48 +building GPT model ... +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4188125184 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight + module.decoder.final_layernorm.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.20.input_layernorm.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.1.self_attention.linear_q_proj.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.linear_fc2.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc1.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.input_layernorm.weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.decoder.layers.0.self_attention.linear_proj.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:20:52 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:20:52 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3424.36, 3436.84) + train/valid/test-data-iterators-setup ..........: (73.14, 92.94) +[before the start of training step] datetime: 2026-08-26 19:20:52 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv index c9c7b5893cb73d190ed97a2f9a544df263ab443c..9e3338ec6f7e9c5fabfaaea338b8ab71ef90a2b2 100644 --- a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv @@ -22,3 +22,179 @@ 5, 1 6, 1 7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 27 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 819 +2, 819 +3, 819 +4, 819 +5, 819 +6, 819 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2551 +1, 2551 +2, 2551 +3, 2551 +4, 2551 +5, 2551 +6, 2551 +7, 2551 +0, 2959 +1, 2959 +2, 2959 +3, 2959 +4, 2959 +5, 2959 +6, 2959 +7, 2959 +0, 51537 +1, 51537 +2, 51537 +3, 51537 +4, 51249 +5, 51537 +6, 51537 +7, 51537 +0, 53019 +1, 53019 +2, 53017 +3, 53019 +4, 52729 +5, 53019 +6, 53017 +7, 53019 +0, 54651 +1, 54651 +2, 54651 +3, 54651 +4, 54363 +5, 54651 +6, 54651 +7, 54651 +0, 55163 +1, 55163 +2, 55163 +3, 55163 +4, 54875 +5, 55163 +6, 55163 +7, 55163 +0, 55163 +1, 55163 +2, 55163 +3, 55163 +4, 54875 +5, 55163 +6, 55163 +7, 55163 +0, 56699 +1, 56699 +2, 56699 +3, 56699 +4, 56411 +5, 56699 +6, 56699 +7, 56699 +0, 87805 +1, 87845 +2, 87845 +3, 87845 +4, 87841 +5, 87845 +6, 87845 +7, 87845 +0, 88517 +1, 88557 +2, 88557 +3, 88557 +4, 88553 +5, 88557 +6, 88557 +7, 88557 +0, 88517 +1, 88557 +2, 88557 +3, 88557 +4, 88553 +5, 88557 +6, 88557 +7, 88557 +0, 88517 +1, 88557 +2, 88557 +3, 88557 +4, 88553 +5, 88557 +6, 88557 +7, 88557 +0, 88921 +1, 88961 +2, 88961 +3, 88961 +4, 88957 +5, 88961 +6, 88961 +7, 88961 +0, 95725 +1, 95789 +2, 95721 +3, 95743 +4, 95923 +5, 95779 +6, 95741 +7, 95759 +0, 111083 +1, 113027 +2, 114123 +3, 111903 +4, 110463 +5, 116199 +6, 108963 +7, 111075 +0, 89083 +1, 89237 +2, 87617 +3, 92987 +4, 91423 +5, 90119 +6, 90181 +7, 88643 +0, 121025 +1, 121181 +2, 121181 +3, 121181 +4, 121321 +5, 121181 +6, 121181 +7, 121181 +0, 5275 +1, 3678 +2, 3678 +3, 3678 +4, 3678 +5, 3678 +6, 3998 +7, 121179 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191143969332Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191143969332Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9877aba5233c8bf8ca087938082e2b277bc0b482 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191143969332Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77298383360, + "active_bytes_current_delta": 32777583104, + "active_bytes_peak": 82477897216, + "active_bytes_peak_delta": 37931931136, + "after": { + "active_bytes_current": 77298383360, + "active_bytes_peak": 82477897216, + "allocated_bytes_current": 77298383360, + "allocated_bytes_peak": 82477897216, + "device": 0, + "inactive_split_bytes_current": 1206404608, + "inactive_split_bytes_peak": 9362196992, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 90250936320, + "reserved_bytes_peak": 90250936320, + "segment_count_current": 1714, + "segment_count_peak": 1714, + "timestamp_utc": "2026-08-26T19:11:43.968857Z", + "world_size": 8 + }, + "allocated_bytes_current": 77298383360, + "allocated_bytes_current_delta": 32777583104, + "allocated_bytes_peak": 82477897216, + "allocated_bytes_peak_delta": 37931931136, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 0, + "inactive_split_bytes_current": 446332928, + "inactive_split_bytes_peak": 1050312704, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:11:17.937284Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1206404608, + "inactive_split_bytes_current_delta": 760071680, + "inactive_split_bytes_peak": 9362196992, + "inactive_split_bytes_peak_delta": 8311884288, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 90250936320, + "reserved_bytes_current_delta": 41999663104, + "reserved_bytes_peak": 90250936320, + "reserved_bytes_peak_delta": 41999663104, + "segment_count_current": 1714, + "segment_count_current_delta": 1093, + "segment_count_peak": 1714, + "segment_count_peak_delta": 1093, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T191143969332Z.pickle", + "timestamp_utc": "2026-08-26T19:11:44.009882Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191143969332Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191143969332Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d9373377da50cc8518992d6bc4bda5159444b820 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191143969332Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:366506f1eecdad1148894ecfb8f4a0105384eba9793cdc997f78dc0f25a97f89 +size 2316617 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191144855486Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191144855486Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4f1c0ba92fe2cce5fa6a42362a7daf2adcf3a214 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191144855486Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77298383360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113343831040, + "active_bytes_peak_delta": 30865933824, + "after": { + "active_bytes_current": 77298383360, + "active_bytes_peak": 113343831040, + "allocated_bytes_current": 77298383360, + "allocated_bytes_peak": 113343831040, + "device": 0, + "inactive_split_bytes_current": 1206404608, + "inactive_split_bytes_peak": 9362196992, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115242696704, + "reserved_bytes_peak": 115242696704, + "segment_count_current": 1923, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T19:11:44.855042Z", + "world_size": 8 + }, + "allocated_bytes_current": 77298383360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113343831040, + "allocated_bytes_peak_delta": 30865933824, + "before": { + "active_bytes_current": 77298383360, + "active_bytes_peak": 82477897216, + "allocated_bytes_current": 77298383360, + "allocated_bytes_peak": 82477897216, + "device": 0, + "inactive_split_bytes_current": 1206404608, + "inactive_split_bytes_peak": 9362196992, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 90250936320, + "reserved_bytes_peak": 90250936320, + "segment_count_current": 1714, + "segment_count_peak": 1714, + "timestamp_utc": "2026-08-26T19:11:44.014581Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1206404608, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9362196992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115242696704, + "reserved_bytes_current_delta": 24991760384, + "reserved_bytes_peak": 115242696704, + "reserved_bytes_peak_delta": 24991760384, + "segment_count_current": 1923, + "segment_count_current_delta": 209, + "segment_count_peak": 1923, + "segment_count_peak_delta": 209, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T191144855486Z.pickle", + "timestamp_utc": "2026-08-26T19:11:44.891762Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191144855486Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191144855486Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d70cdf8ab0508a444fdae53f71f1594eea0f7133 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191144855486Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9d644fd17905670a38a6b94b76cb0b9e6d9eeb7d69c1392488c0adbc3ebceca +size 1844220 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191145209517Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191145209517Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9d1953a79ce809c5dd0aae96584151d50532b429 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191145209517Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77298383360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113343831040, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77298383360, + "active_bytes_peak": 113343831040, + "allocated_bytes_current": 77298383360, + "allocated_bytes_peak": 113343831040, + "device": 0, + "inactive_split_bytes_current": 1206404608, + "inactive_split_bytes_peak": 9362196992, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115242696704, + "reserved_bytes_peak": 115242696704, + "segment_count_current": 1923, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T19:11:45.209078Z", + "world_size": 8 + }, + "allocated_bytes_current": 77298383360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113343831040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77298383360, + "active_bytes_peak": 113343831040, + "allocated_bytes_current": 77298383360, + "allocated_bytes_peak": 113343831040, + "device": 0, + "inactive_split_bytes_current": 1206404608, + "inactive_split_bytes_peak": 9362196992, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115242696704, + "reserved_bytes_peak": 115242696704, + "segment_count_current": 1923, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T19:11:44.895755Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1206404608, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9362196992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115242696704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115242696704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1923, + "segment_count_current_delta": 0, + "segment_count_peak": 1923, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T191145209517Z.pickle", + "timestamp_utc": "2026-08-26T19:11:45.240454Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191145209517Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191145209517Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44f25c4e328d510b70b60b35b5ebc0d917c1bac1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191145209517Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac7edbe46c6155a1d9a064c5c24884e487feb90b7fc1401f2f098c7baecb476d +size 1839099 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191147859661Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191147859661Z.json new file mode 100644 index 0000000000000000000000000000000000000000..66b176ced05608dfe11e9d4bdd6ab739393e3756 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191147859661Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77315426816, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 113351470080, + "active_bytes_peak_delta": 7639040, + "after": { + "active_bytes_current": 77315426816, + "active_bytes_peak": 113351470080, + "allocated_bytes_current": 77315426816, + "allocated_bytes_peak": 113351470080, + "device": 0, + "inactive_split_bytes_current": 1315190272, + "inactive_split_bytes_peak": 10511450112, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116230455296, + "reserved_bytes_peak": 116230455296, + "segment_count_current": 1696, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T19:11:47.859107Z", + "world_size": 8 + }, + "allocated_bytes_current": 77315426816, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113351470080, + "allocated_bytes_peak_delta": 7639040, + "before": { + "active_bytes_current": 77298383360, + "active_bytes_peak": 113343831040, + "allocated_bytes_current": 77298383360, + "allocated_bytes_peak": 113343831040, + "device": 0, + "inactive_split_bytes_current": 1206404608, + "inactive_split_bytes_peak": 9362196992, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115242696704, + "reserved_bytes_peak": 115242696704, + "segment_count_current": 1923, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T19:11:45.244071Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1315190272, + "inactive_split_bytes_current_delta": 108785664, + "inactive_split_bytes_peak": 10511450112, + "inactive_split_bytes_peak_delta": 1149253120, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116230455296, + "reserved_bytes_current_delta": 987758592, + "reserved_bytes_peak": 116230455296, + "reserved_bytes_peak_delta": 987758592, + "segment_count_current": 1696, + "segment_count_current_delta": -227, + "segment_count_peak": 1923, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T191147859661Z.pickle", + "timestamp_utc": "2026-08-26T19:11:47.897560Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191147859661Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191147859661Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5b3e1957eb08b4c8896b56842c04b5bc977c3aa9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191147859661Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:189e0599ddd5f751d4966b5bc19cf04a1f32edf44107bc89f91ca9060c9bee95 +size 2181131 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191148200823Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191148200823Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bdea3bb0255b2c40d225d077fd7a91084ed04b01 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191148200823Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77315426816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113351470080, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77315426816, + "active_bytes_peak": 113351470080, + "allocated_bytes_current": 77315426816, + "allocated_bytes_peak": 113351470080, + "device": 0, + "inactive_split_bytes_current": 1315190272, + "inactive_split_bytes_peak": 10511450112, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116230455296, + "reserved_bytes_peak": 116230455296, + "segment_count_current": 1696, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T19:11:48.200437Z", + "world_size": 8 + }, + "allocated_bytes_current": 77315426816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113351470080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77315426816, + "active_bytes_peak": 113351470080, + "allocated_bytes_current": 77315426816, + "allocated_bytes_peak": 113351470080, + "device": 0, + "inactive_split_bytes_current": 1315190272, + "inactive_split_bytes_peak": 10511450112, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116230455296, + "reserved_bytes_peak": 116230455296, + "segment_count_current": 1696, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T19:11:47.901776Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1315190272, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10511450112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116230455296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116230455296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1696, + "segment_count_current_delta": 0, + "segment_count_peak": 1923, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T191148200823Z.pickle", + "timestamp_utc": "2026-08-26T19:11:48.209298Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191148200823Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191148200823Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..11b74bec53fd3e9bb35d97fcd792f2351d209258 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191148200823Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eecba693a4d72b91d813578a83b637d48f90ddfe5cfcbd277eecd5701aa4ce2 +size 418854 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191148509603Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191148509603Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ecf6ae20f578baf9ea6b2a5855d54b89298459f0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191148509603Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77315426816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113351470080, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77315426816, + "active_bytes_peak": 113351470080, + "allocated_bytes_current": 77315426816, + "allocated_bytes_peak": 113351470080, + "device": 0, + "inactive_split_bytes_current": 1315190272, + "inactive_split_bytes_peak": 10511450112, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116230455296, + "reserved_bytes_peak": 116230455296, + "segment_count_current": 1696, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T19:11:48.509226Z", + "world_size": 8 + }, + "allocated_bytes_current": 77315426816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113351470080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77315426816, + "active_bytes_peak": 113351470080, + "allocated_bytes_current": 77315426816, + "allocated_bytes_peak": 113351470080, + "device": 0, + "inactive_split_bytes_current": 1315190272, + "inactive_split_bytes_peak": 10511450112, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116230455296, + "reserved_bytes_peak": 116230455296, + "segment_count_current": 1696, + "segment_count_peak": 1923, + "timestamp_utc": "2026-08-26T19:11:48.210418Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1315190272, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10511450112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116230455296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116230455296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1696, + "segment_count_current_delta": 0, + "segment_count_peak": 1923, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T191148509603Z.pickle", + "timestamp_utc": "2026-08-26T19:11:48.518016Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191148509603Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191148509603Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3d3febb085bccc1176b1138a8a892c4affc59b03 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191148509603Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6184337a698a26e0528082a885307cd77aaee32329aa3ad51f78c8bc74eb1ac +size 418854 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191143969245Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191143969245Z.json new file mode 100644 index 0000000000000000000000000000000000000000..18d04828a2502d4232a330cd58ae2069346a1266 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191143969245Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 32778758656, + "active_bytes_peak": 82480875008, + "active_bytes_peak_delta": 37934908928, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 82480875008, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 82480875008, + "device": 1, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 90267713536, + "reserved_bytes_peak": 90267713536, + "segment_count_current": 1718, + "segment_count_peak": 1718, + "timestamp_utc": "2026-08-26T19:11:43.968739Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 32778758656, + "allocated_bytes_peak": 82480875008, + "allocated_bytes_peak_delta": 37934908928, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 1, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:11:17.937156Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 947639808, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 8523717120, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 90267713536, + "reserved_bytes_current_delta": 42016440320, + "reserved_bytes_peak": 90267713536, + "reserved_bytes_peak_delta": 42016440320, + "segment_count_current": 1718, + "segment_count_current_delta": 1097, + "segment_count_peak": 1718, + "segment_count_peak_delta": 1097, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T191143969245Z.pickle", + "timestamp_utc": "2026-08-26T19:11:44.009111Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191143969245Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191143969245Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f9f63900886ad7bd1984fdf4c404889604b27d65 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191143969245Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd63bf0418f543d422370e8d8d4fe5ffdc6f159df955028160cd3a2d0c88c29f +size 2304358 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191144855476Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191144855476Z.json new file mode 100644 index 0000000000000000000000000000000000000000..55236ee2fde8b267e5efa8a2fbea6d6220833260 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191144855476Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113344452608, + "active_bytes_peak_delta": 30863577600, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 1, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:44.855039Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113344452608, + "allocated_bytes_peak_delta": 30863577600, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 82480875008, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 82480875008, + "device": 1, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 90267713536, + "reserved_bytes_peak": 90267713536, + "segment_count_current": 1718, + "segment_count_peak": 1718, + "timestamp_utc": "2026-08-26T19:11:44.013716Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 115364331520, + "reserved_bytes_current_delta": 25096617984, + "reserved_bytes_peak": 115364331520, + "reserved_bytes_peak_delta": 25096617984, + "segment_count_current": 1934, + "segment_count_current_delta": 216, + "segment_count_peak": 1934, + "segment_count_peak_delta": 216, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T191144855476Z.pickle", + "timestamp_utc": "2026-08-26T19:11:44.892389Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191144855476Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191144855476Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ccd87eb6d303c664123d1312208b17e8df7daf75 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191144855476Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecc694e58f210966e555161d3a63de0fdffbbb234ab434109bef0b153934e320 +size 1833396 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191145209458Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191145209458Z.json new file mode 100644 index 0000000000000000000000000000000000000000..829c639a06e557db4d295f4f0d51912794564afa --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191145209458Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113344452608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 1, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:45.209053Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113344452608, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 1, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:44.896325Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 115364331520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115364331520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1934, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T191145209458Z.pickle", + "timestamp_utc": "2026-08-26T19:11:45.239689Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191145209458Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191145209458Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a4889d9168d59e7dc71477cf1769a7b6385e9117 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191145209458Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb9fe89f6b89e5abf53df921b3b39cce8a047024604744279461e1fd96a10be0 +size 1828450 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191147859527Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191147859527Z.json new file mode 100644 index 0000000000000000000000000000000000000000..472d04b4e315bba3e3f973a9f81a79d3d8140fe2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191147859527Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 8193024, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 1, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:47.858997Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 8193024, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 1, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:45.243253Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 108785664, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 1124988416, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 1029701632, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 1029701632, + "segment_count_current": 1701, + "segment_count_current_delta": -233, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T191147859527Z.pickle", + "timestamp_utc": "2026-08-26T19:11:47.897085Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191147859527Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191147859527Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f5dc6433dbcc8b1d0b4af00fbb95042ec96b8562 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191147859527Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45316b56ee65848ab6595cb2c9fe9646bc514f9e4526201f8bd7952224ce759a +size 2193727 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191148200859Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191148200859Z.json new file mode 100644 index 0000000000000000000000000000000000000000..02d76f63356b237f72257cfc6adf83f01dd68b42 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191148200859Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 1, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.200462Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 1, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:47.901088Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1701, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T191148200859Z.pickle", + "timestamp_utc": "2026-08-26T19:11:48.209181Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191148200859Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191148200859Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e33f8028f9c2bb64b7054a983f2f12860edec1d3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191148200859Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05272f421f21a224b022b0c8f2193e5cd7d2b5f296dee60b14863343931dce7b +size 411770 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191148509603Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191148509603Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b0874989b24fbd6ff8541b2fbdfd20004b992d0a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191148509603Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 1, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.509218Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 1, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.210270Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1701, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T191148509603Z.pickle", + "timestamp_utc": "2026-08-26T19:11:48.518027Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191148509603Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191148509603Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c4bf962cbbb22556dd9595c611d5b54d2b051ff6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191148509603Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc4c76682af535734ff96acf66ab2758651d9119bab6ea704c4c024e0cab9e14 +size 411770 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191143969307Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191143969307Z.json new file mode 100644 index 0000000000000000000000000000000000000000..12b9d81b2296733cabaf67e2e52125e96b1f55d4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191143969307Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 32778758656, + "active_bytes_peak": 82480875008, + "active_bytes_peak_delta": 37934908928, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 82480875008, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 82480875008, + "device": 2, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 90267713536, + "reserved_bytes_peak": 90267713536, + "segment_count_current": 1718, + "segment_count_peak": 1718, + "timestamp_utc": "2026-08-26T19:11:43.968843Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 32778758656, + "allocated_bytes_peak": 82480875008, + "allocated_bytes_peak_delta": 37934908928, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 2, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:11:17.937342Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 947639808, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 8523717120, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 90267713536, + "reserved_bytes_current_delta": 42016440320, + "reserved_bytes_peak": 90267713536, + "reserved_bytes_peak_delta": 42016440320, + "segment_count_current": 1718, + "segment_count_current_delta": 1097, + "segment_count_peak": 1718, + "segment_count_peak_delta": 1097, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T191143969307Z.pickle", + "timestamp_utc": "2026-08-26T19:11:44.009608Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191143969307Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191143969307Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..90cf21c6294bbdfa3d54d23839229bb13718d0ab --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191143969307Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:815384b03107533a05df19ec1085b9480eefa54d8bd0488516d73ea922bb3ecd +size 2304358 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191144855609Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191144855609Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bbac41d8475d482b9ef4f5fc486c0a4e04f00c8d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191144855609Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113344452608, + "active_bytes_peak_delta": 30863577600, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 2, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:44.855177Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113344452608, + "allocated_bytes_peak_delta": 30863577600, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 82480875008, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 82480875008, + "device": 2, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 90267713536, + "reserved_bytes_peak": 90267713536, + "segment_count_current": 1718, + "segment_count_peak": 1718, + "timestamp_utc": "2026-08-26T19:11:44.014261Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 115364331520, + "reserved_bytes_current_delta": 25096617984, + "reserved_bytes_peak": 115364331520, + "reserved_bytes_peak_delta": 25096617984, + "segment_count_current": 1934, + "segment_count_current_delta": 216, + "segment_count_peak": 1934, + "segment_count_peak_delta": 216, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T191144855609Z.pickle", + "timestamp_utc": "2026-08-26T19:11:44.892180Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191144855609Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191144855609Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d6bce4a5f5eca306f21ae96ee22def94bc473135 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191144855609Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0e924ed7d72f88dd565acd94a3993cca8869ec8eaf1d717b05d561c9cf5939f +size 1833399 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191145209564Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191145209564Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b57241512afefdc325b2bf14155077ed05d6d112 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191145209564Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113344452608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 2, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:45.209140Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113344452608, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 2, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:44.896151Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 115364331520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115364331520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1934, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T191145209564Z.pickle", + "timestamp_utc": "2026-08-26T19:11:45.240203Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191145209564Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191145209564Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d3a79229a41dacec471660ba7601b740313f8144 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191145209564Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d8f6f12368938c04a7a33415f661c3e6e261df0a86be8fe035119b6f60a55ce +size 1828562 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191147859711Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191147859711Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9c149d402fce8c3a61aa2b6bd82ff15c939b376f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191147859711Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 8193024, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 2, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:47.859142Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 8193024, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 2, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:45.243780Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 108785664, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 1124988416, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 1029701632, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 1029701632, + "segment_count_current": 1701, + "segment_count_current_delta": -233, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T191147859711Z.pickle", + "timestamp_utc": "2026-08-26T19:11:47.897687Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191147859711Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191147859711Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..25256479f3473ca2cfe4c2964c49daee24f9b458 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191147859711Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be9b4543ec95a2ba876ec647c3bf34f42c3c7d64e3aa58f3bc5c440b6f7afeb3 +size 2193889 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191148200943Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191148200943Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0c17244f4b79ddb0fe690f908c2e8df48a1299f3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191148200943Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 2, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.200546Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 2, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:47.902545Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1701, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T191148200943Z.pickle", + "timestamp_utc": "2026-08-26T19:11:48.209332Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191148200943Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191148200943Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f038bf6c76f22c2fa12346caab21423acdad1d9b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191148200943Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3a4d3b1d51744d2bb80d93fbaf542d0cd7ef040974de68a67e6b149c6e0dcdc +size 411608 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191148509664Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191148509664Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e43396fef755d99509f661e0b8eff0a15bf94ae4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191148509664Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 2, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.509293Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 2, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.210429Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1701, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T191148509664Z.pickle", + "timestamp_utc": "2026-08-26T19:11:48.518004Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191148509664Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191148509664Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9e6647437315cb3256139f051a709f7e38f5a730 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191148509664Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:964457ab0632888b1b0e58ff3c1f51b88f659ffb6b3f8e3cbdb133b3f8d503ad +size 411608 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191143969303Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191143969303Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4154db694f48815defcd8f9bacf8b1cdd50db921 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191143969303Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 32778758656, + "active_bytes_peak": 82480875008, + "active_bytes_peak_delta": 37934908928, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 82480875008, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 82480875008, + "device": 3, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 90267713536, + "reserved_bytes_peak": 90267713536, + "segment_count_current": 1718, + "segment_count_peak": 1718, + "timestamp_utc": "2026-08-26T19:11:43.968838Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 32778758656, + "allocated_bytes_peak": 82480875008, + "allocated_bytes_peak_delta": 37934908928, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 3, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:11:17.937038Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 947639808, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 8523717120, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 90267713536, + "reserved_bytes_current_delta": 42016440320, + "reserved_bytes_peak": 90267713536, + "reserved_bytes_peak_delta": 42016440320, + "segment_count_current": 1718, + "segment_count_current_delta": 1097, + "segment_count_peak": 1718, + "segment_count_peak_delta": 1097, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T191143969303Z.pickle", + "timestamp_utc": "2026-08-26T19:11:44.008245Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191143969303Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191143969303Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..47458956f52e11b696bf21db1c4ffc379600fc8c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191143969303Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f8d9f6081984ccd5c5fe51aab0a3e3e8467fdc958903e68857b52025e995c65 +size 2304358 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191144855619Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191144855619Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5f44cda5eb6d6bfa16f6c02ada432ad9a01a2a8b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191144855619Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113344452608, + "active_bytes_peak_delta": 30863577600, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 3, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:44.855203Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113344452608, + "allocated_bytes_peak_delta": 30863577600, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 82480875008, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 82480875008, + "device": 3, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 90267713536, + "reserved_bytes_peak": 90267713536, + "segment_count_current": 1718, + "segment_count_peak": 1718, + "timestamp_utc": "2026-08-26T19:11:44.012798Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 115364331520, + "reserved_bytes_current_delta": 25096617984, + "reserved_bytes_peak": 115364331520, + "reserved_bytes_peak_delta": 25096617984, + "segment_count_current": 1934, + "segment_count_current_delta": 216, + "segment_count_peak": 1934, + "segment_count_peak_delta": 216, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T191144855619Z.pickle", + "timestamp_utc": "2026-08-26T19:11:44.892124Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191144855619Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191144855619Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..482fd8bad1075e2f7177d022c780bc3d45588526 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191144855619Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e16105fc8a481ebf77593be97dde94d82e8367120bc404653621454fbc513eeb +size 1833396 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191145209541Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191145209541Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b80b73d5dbfc8f213f041db95be73d236b93f940 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191145209541Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113344452608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 3, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:45.209139Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113344452608, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 3, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:44.896027Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 115364331520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115364331520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1934, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T191145209541Z.pickle", + "timestamp_utc": "2026-08-26T19:11:45.240436Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191145209541Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191145209541Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9c210df285577e6d5c158c3d3fdb5a32c1696156 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191145209541Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04cad6d6b0d89aa3a18cdbe7ae40d41a1b82d05deda56a0d48972a25f5405133 +size 1828450 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191147859681Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191147859681Z.json new file mode 100644 index 0000000000000000000000000000000000000000..32a2270efac3855306d2e4c14ab351f7cff8688c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191147859681Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 8193024, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 3, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:47.859143Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 8193024, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 3, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:45.243981Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 108785664, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 1124988416, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 1029701632, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 1029701632, + "segment_count_current": 1701, + "segment_count_current_delta": -233, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T191147859681Z.pickle", + "timestamp_utc": "2026-08-26T19:11:47.897322Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191147859681Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191147859681Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e66760a245889904760512568e968406a57fca9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191147859681Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e52652eb4021b76676884a199a82ac80943346346a366f19bfbcad71fabb5b7 +size 2193725 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191148200939Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191148200939Z.json new file mode 100644 index 0000000000000000000000000000000000000000..01ede4fcfc02fbc83bad071eaeb8d887b5f65b72 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191148200939Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 3, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.200547Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 3, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:47.901458Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1701, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T191148200939Z.pickle", + "timestamp_utc": "2026-08-26T19:11:48.209259Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191148200939Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191148200939Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..078f28f9e61e3f237660291a70a7137b19fbd79b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191148200939Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe460f29325376f03ccea0720aacafe5fe7c016cd04f88b0a95e2395321f6b41 +size 411768 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191148509684Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191148509684Z.json new file mode 100644 index 0000000000000000000000000000000000000000..88b7fd4e3dda2d514a5efbfd8a4addbdb850bf4c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191148509684Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 3, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.509298Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 3, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.210347Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1701, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T191148509684Z.pickle", + "timestamp_utc": "2026-08-26T19:11:48.518070Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191148509684Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191148509684Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3399b04d6dbf5b7743b13a086f9b88523a0becb9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191148509684Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71aab16b8d6d4f2a19ac0d229ff8e4acf0f20ffcde810ed871a22e61608ff4eb +size 411768 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191143969247Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191143969247Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8671f5406c99321820d40a59db008a3be387cc0f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191143969247Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77300942336, + "active_bytes_current_delta": 32780142080, + "active_bytes_peak": 82490928640, + "active_bytes_peak_delta": 37944962560, + "after": { + "active_bytes_current": 77300942336, + "active_bytes_peak": 82490928640, + "allocated_bytes_current": 77300942336, + "allocated_bytes_peak": 82490928640, + "device": 4, + "inactive_split_bytes_current": 1512126976, + "inactive_split_bytes_peak": 9827441664, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 90364182528, + "reserved_bytes_peak": 90364182528, + "segment_count_current": 1729, + "segment_count_peak": 1729, + "timestamp_utc": "2026-08-26T19:11:43.968725Z", + "world_size": 8 + }, + "allocated_bytes_current": 77300942336, + "allocated_bytes_current_delta": 32780142080, + "allocated_bytes_peak": 82490928640, + "allocated_bytes_peak_delta": 37944962560, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 4, + "inactive_split_bytes_current": 270172160, + "inactive_split_bytes_peak": 874151936, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 47949283328, + "reserved_bytes_peak": 47949283328, + "segment_count_current": 615, + "segment_count_peak": 615, + "timestamp_utc": "2026-08-26T19:11:17.937327Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1512126976, + "inactive_split_bytes_current_delta": 1241954816, + "inactive_split_bytes_peak": 9827441664, + "inactive_split_bytes_peak_delta": 8953289728, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 90364182528, + "reserved_bytes_current_delta": 42414899200, + "reserved_bytes_peak": 90364182528, + "reserved_bytes_peak_delta": 42414899200, + "segment_count_current": 1729, + "segment_count_current_delta": 1114, + "segment_count_peak": 1729, + "segment_count_peak_delta": 1114, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T191143969247Z.pickle", + "timestamp_utc": "2026-08-26T19:11:44.009786Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191143969247Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191143969247Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b3a3c2c1a3bf5577bb723d7f530e9b26b43d6de6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191143969247Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec97dad9c4236b28c8fa204793de8928a5059e27e47fb8cc7a735bad95931654 +size 2313033 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191144855463Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191144855463Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d202dcad0bf1b0c33f536587802433f75ad74cd --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191144855463Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77300942336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113382964736, + "active_bytes_peak_delta": 30892036096, + "after": { + "active_bytes_current": 77300942336, + "active_bytes_peak": 113382964736, + "allocated_bytes_current": 77300942336, + "allocated_bytes_peak": 113382964736, + "device": 4, + "inactive_split_bytes_current": 1514224128, + "inactive_split_bytes_peak": 9827441664, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115414663168, + "reserved_bytes_peak": 115414663168, + "segment_count_current": 1942, + "segment_count_peak": 1942, + "timestamp_utc": "2026-08-26T19:11:44.855029Z", + "world_size": 8 + }, + "allocated_bytes_current": 77300942336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113382964736, + "allocated_bytes_peak_delta": 30892036096, + "before": { + "active_bytes_current": 77300942336, + "active_bytes_peak": 82490928640, + "allocated_bytes_current": 77300942336, + "allocated_bytes_peak": 82490928640, + "device": 4, + "inactive_split_bytes_current": 1512126976, + "inactive_split_bytes_peak": 9827441664, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 90364182528, + "reserved_bytes_peak": 90364182528, + "segment_count_current": 1729, + "segment_count_peak": 1729, + "timestamp_utc": "2026-08-26T19:11:44.014312Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1514224128, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 9827441664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115414663168, + "reserved_bytes_current_delta": 25050480640, + "reserved_bytes_peak": 115414663168, + "reserved_bytes_peak_delta": 25050480640, + "segment_count_current": 1942, + "segment_count_current_delta": 213, + "segment_count_peak": 1942, + "segment_count_peak_delta": 213, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T191144855463Z.pickle", + "timestamp_utc": "2026-08-26T19:11:44.892630Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191144855463Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191144855463Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a82c01a5eea21bfcc173a24357cdf73662124e1a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191144855463Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73f71eadc586b99ad74de367a7f5befc01800e0a19e95e51d60f08c02338d82a +size 1838401 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191145209418Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191145209418Z.json new file mode 100644 index 0000000000000000000000000000000000000000..538f5b4fd7e6a0ed1ed99ea32bc7834768ff2802 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191145209418Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77300942336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113389122560, + "active_bytes_peak_delta": 6157824, + "after": { + "active_bytes_current": 77300942336, + "active_bytes_peak": 113389122560, + "allocated_bytes_current": 77300942336, + "allocated_bytes_peak": 113389122560, + "device": 4, + "inactive_split_bytes_current": 1514224128, + "inactive_split_bytes_peak": 9827441664, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115414663168, + "reserved_bytes_peak": 115414663168, + "segment_count_current": 1942, + "segment_count_peak": 1942, + "timestamp_utc": "2026-08-26T19:11:45.209018Z", + "world_size": 8 + }, + "allocated_bytes_current": 77300942336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113389122560, + "allocated_bytes_peak_delta": 6157824, + "before": { + "active_bytes_current": 77300942336, + "active_bytes_peak": 113382964736, + "allocated_bytes_current": 77300942336, + "allocated_bytes_peak": 113382964736, + "device": 4, + "inactive_split_bytes_current": 1514224128, + "inactive_split_bytes_peak": 9827441664, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115414663168, + "reserved_bytes_peak": 115414663168, + "segment_count_current": 1942, + "segment_count_peak": 1942, + "timestamp_utc": "2026-08-26T19:11:44.896597Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1514224128, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9827441664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115414663168, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115414663168, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1942, + "segment_count_current_delta": 0, + "segment_count_peak": 1942, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T191145209418Z.pickle", + "timestamp_utc": "2026-08-26T19:11:45.240917Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191145209418Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191145209418Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8a43774d5f9ced03225694f2642f6ddf73c5c9df --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191145209418Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c707af2d1d203a9a46b37f64836812418241a2fe07bafa637d40aaa826fe933a +size 1876700 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191147859516Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191147859516Z.json new file mode 100644 index 0000000000000000000000000000000000000000..34b75146a240e5464266e5507c232aa6fb154f01 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191147859516Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77317985792, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 113389122560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77317985792, + "active_bytes_peak": 113389122560, + "allocated_bytes_current": 77317985792, + "allocated_bytes_peak": 113389122560, + "device": 4, + "inactive_split_bytes_current": 1620912640, + "inactive_split_bytes_peak": 10819269632, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116540833792, + "reserved_bytes_peak": 116540833792, + "segment_count_current": 1730, + "segment_count_peak": 1942, + "timestamp_utc": "2026-08-26T19:11:47.858989Z", + "world_size": 8 + }, + "allocated_bytes_current": 77317985792, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113389122560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77300942336, + "active_bytes_peak": 113389122560, + "allocated_bytes_current": 77300942336, + "allocated_bytes_peak": 113389122560, + "device": 4, + "inactive_split_bytes_current": 1514224128, + "inactive_split_bytes_peak": 9827441664, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115414663168, + "reserved_bytes_peak": 115414663168, + "segment_count_current": 1942, + "segment_count_peak": 1942, + "timestamp_utc": "2026-08-26T19:11:45.244424Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1620912640, + "inactive_split_bytes_current_delta": 106688512, + "inactive_split_bytes_peak": 10819269632, + "inactive_split_bytes_peak_delta": 991827968, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116540833792, + "reserved_bytes_current_delta": 1126170624, + "reserved_bytes_peak": 116540833792, + "reserved_bytes_peak_delta": 1126170624, + "segment_count_current": 1730, + "segment_count_current_delta": -212, + "segment_count_peak": 1942, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T191147859516Z.pickle", + "timestamp_utc": "2026-08-26T19:11:47.897518Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191147859516Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191147859516Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..42e4ceae3be4bf917a82e9c0f1c81746a5015bbe --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191147859516Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db0bc85972cff0be79638b061382b3016564c1680c1e10d07eabd55e1c677c21 +size 2175945 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191148200936Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191148200936Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f6b91ed15817ef9d463c8366b4481fe635d13942 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191148200936Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77317985792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113389122560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77317985792, + "active_bytes_peak": 113389122560, + "allocated_bytes_current": 77317985792, + "allocated_bytes_peak": 113389122560, + "device": 4, + "inactive_split_bytes_current": 1620912640, + "inactive_split_bytes_peak": 10819269632, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116540833792, + "reserved_bytes_peak": 116540833792, + "segment_count_current": 1730, + "segment_count_peak": 1942, + "timestamp_utc": "2026-08-26T19:11:48.200471Z", + "world_size": 8 + }, + "allocated_bytes_current": 77317985792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113389122560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77317985792, + "active_bytes_peak": 113389122560, + "allocated_bytes_current": 77317985792, + "allocated_bytes_peak": 113389122560, + "device": 4, + "inactive_split_bytes_current": 1620912640, + "inactive_split_bytes_peak": 10819269632, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116540833792, + "reserved_bytes_peak": 116540833792, + "segment_count_current": 1730, + "segment_count_peak": 1942, + "timestamp_utc": "2026-08-26T19:11:47.901678Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1620912640, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10819269632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116540833792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116540833792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1730, + "segment_count_current_delta": 0, + "segment_count_peak": 1942, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T191148200936Z.pickle", + "timestamp_utc": "2026-08-26T19:11:48.209457Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191148200936Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191148200936Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7f8430c56e65a40cb555f36bd28b24af94434598 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191148200936Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3089321fbb411a036c6d9f99dd2beadee5af84b6bb770c29f4ed54e701c34d6 +size 416807 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191148509559Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191148509559Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0850fe87c92649a465f69c1a8bbdc66d451ce80c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191148509559Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77317985792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113389122560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77317985792, + "active_bytes_peak": 113389122560, + "allocated_bytes_current": 77317985792, + "allocated_bytes_peak": 113389122560, + "device": 4, + "inactive_split_bytes_current": 1620912640, + "inactive_split_bytes_peak": 10819269632, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116540833792, + "reserved_bytes_peak": 116540833792, + "segment_count_current": 1730, + "segment_count_peak": 1942, + "timestamp_utc": "2026-08-26T19:11:48.509171Z", + "world_size": 8 + }, + "allocated_bytes_current": 77317985792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113389122560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77317985792, + "active_bytes_peak": 113389122560, + "allocated_bytes_current": 77317985792, + "allocated_bytes_peak": 113389122560, + "device": 4, + "inactive_split_bytes_current": 1620912640, + "inactive_split_bytes_peak": 10819269632, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116540833792, + "reserved_bytes_peak": 116540833792, + "segment_count_current": 1730, + "segment_count_peak": 1942, + "timestamp_utc": "2026-08-26T19:11:48.210556Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1620912640, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10819269632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116540833792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116540833792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1730, + "segment_count_current_delta": 0, + "segment_count_peak": 1942, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T191148509559Z.pickle", + "timestamp_utc": "2026-08-26T19:11:48.517947Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191148509559Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191148509559Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dca27f2503b56fdef55ac959a099f6a5bcc6eefe --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191148509559Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db719b89d9270346a729b47e9bb29b5bee4ee251733038a4e50600d866583886 +size 416807 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191143969232Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191143969232Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4fc89f388c10ae367532493233db14edc2008794 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191143969232Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 32778758656, + "active_bytes_peak": 82480875008, + "active_bytes_peak_delta": 37934908928, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 82480875008, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 82480875008, + "device": 5, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 90267713536, + "reserved_bytes_peak": 90267713536, + "segment_count_current": 1718, + "segment_count_peak": 1718, + "timestamp_utc": "2026-08-26T19:11:43.968716Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 32778758656, + "allocated_bytes_peak": 82480875008, + "allocated_bytes_peak_delta": 37934908928, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 5, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:11:17.937122Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 947639808, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 8523717120, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 90267713536, + "reserved_bytes_current_delta": 42016440320, + "reserved_bytes_peak": 90267713536, + "reserved_bytes_peak_delta": 42016440320, + "segment_count_current": 1718, + "segment_count_current_delta": 1097, + "segment_count_peak": 1718, + "segment_count_peak_delta": 1097, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T191143969232Z.pickle", + "timestamp_utc": "2026-08-26T19:11:44.009129Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191143969232Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191143969232Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5fa8e6024589067dcba1dffd7d026f7d403a23f0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191143969232Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f73e3afe1ea1616df127698c9505335b8508269189f060a6619f8cd4ecffeb15 +size 2304358 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191144855464Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191144855464Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ebce2778aa2dc4a15c2d9e21195e13e84b6bd413 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191144855464Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113344452608, + "active_bytes_peak_delta": 30863577600, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 5, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:44.855026Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113344452608, + "allocated_bytes_peak_delta": 30863577600, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 82480875008, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 82480875008, + "device": 5, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 90267713536, + "reserved_bytes_peak": 90267713536, + "segment_count_current": 1718, + "segment_count_peak": 1718, + "timestamp_utc": "2026-08-26T19:11:44.013687Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 115364331520, + "reserved_bytes_current_delta": 25096617984, + "reserved_bytes_peak": 115364331520, + "reserved_bytes_peak_delta": 25096617984, + "segment_count_current": 1934, + "segment_count_current_delta": 216, + "segment_count_peak": 1934, + "segment_count_peak_delta": 216, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T191144855464Z.pickle", + "timestamp_utc": "2026-08-26T19:11:44.892280Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191144855464Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191144855464Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cfbc1de0ca5d706f5711dfa1608ae31f3389266a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191144855464Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ba02fe1551e197e6ab4a1c84eac40e3f7ef0a1e3f9dba203bb663e018683d66 +size 1833396 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191145209414Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191145209414Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1ddedc3d2cf3f7f05cc645ced4ca3a5dc9d1cac1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191145209414Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113344452608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 5, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:45.209007Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113344452608, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 5, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:44.896359Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 115364331520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115364331520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1934, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T191145209414Z.pickle", + "timestamp_utc": "2026-08-26T19:11:45.240324Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191145209414Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191145209414Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3f9947d0af13bc55bb6cbcb7393228a35a57f820 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191145209414Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaa4bf8605a885354cbaf6244f8528d730f9055d27e2bd2b94af1eaa75abba2e +size 1828450 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191147859400Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191147859400Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d708eb50c86654803ea88d0689a8171317d70ab3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191147859400Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 8193024, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 5, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:47.858975Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 8193024, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 5, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:45.243926Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 108785664, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 1124988416, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 1029701632, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 1029701632, + "segment_count_current": 1701, + "segment_count_current_delta": -233, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T191147859400Z.pickle", + "timestamp_utc": "2026-08-26T19:11:47.896966Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191147859400Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191147859400Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d2cef2bd07d03d3e7f73b88f8af6f14a48bad4d3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191147859400Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a1d13282af7139c1190bee731ac2bd50fb4fbfc2521785341e1d90ecbadbbcf +size 2193727 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191148200850Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191148200850Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9f9782e160199cfe051164f8965a257e28ac3eba --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191148200850Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 5, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.200471Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 5, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:47.901112Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1701, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T191148200850Z.pickle", + "timestamp_utc": "2026-08-26T19:11:48.209094Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191148200850Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191148200850Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a70a46c10889c8232943bdb101a5ea5a1fafb4db --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191148200850Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:763e33d5a76460ea5427a165d1853acbbc907c98ff1d79a3036f6023176a9db0 +size 411770 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191148509560Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191148509560Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4e82ccf8cd7a27726f63b098045f3347e47a581a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191148509560Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 5, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.509175Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 5, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.210163Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1701, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T191148509560Z.pickle", + "timestamp_utc": "2026-08-26T19:11:48.517762Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191148509560Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191148509560Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2681735dbb8e172309b8b565de81f33f4a9d39d8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191148509560Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e71fe5847b0cbb517db72203fb40fbb500b4ef7f49e38a14ccb652fde47cd65 +size 411770 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191143969327Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191143969327Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1fdd63b8420daf2cc202d3daf44350ee6b59fa2d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191143969327Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 32778758656, + "active_bytes_peak": 82480875008, + "active_bytes_peak_delta": 37934908928, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 82480875008, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 82480875008, + "device": 6, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 90267713536, + "reserved_bytes_peak": 90267713536, + "segment_count_current": 1718, + "segment_count_peak": 1718, + "timestamp_utc": "2026-08-26T19:11:43.968853Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 32778758656, + "allocated_bytes_peak": 82480875008, + "allocated_bytes_peak_delta": 37934908928, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 6, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:11:17.937239Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 947639808, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 8523717120, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 90267713536, + "reserved_bytes_current_delta": 42016440320, + "reserved_bytes_peak": 90267713536, + "reserved_bytes_peak_delta": 42016440320, + "segment_count_current": 1718, + "segment_count_current_delta": 1097, + "segment_count_peak": 1718, + "segment_count_peak_delta": 1097, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T191143969327Z.pickle", + "timestamp_utc": "2026-08-26T19:11:44.009549Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191143969327Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191143969327Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..53bca7ecae7fdaec3876e91e8cac5e095c99ce10 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191143969327Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17b5685a1a207d42cb338cf5bae8a6f260845357da32d9d7ca8dbdc8c572a733 +size 2304358 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191144855658Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191144855658Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15e0cdd1ad81bbed457b200a490c976d734b2348 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191144855658Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113344452608, + "active_bytes_peak_delta": 30863577600, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 6, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:44.855224Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113344452608, + "allocated_bytes_peak_delta": 30863577600, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 82480875008, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 82480875008, + "device": 6, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 90267713536, + "reserved_bytes_peak": 90267713536, + "segment_count_current": 1718, + "segment_count_peak": 1718, + "timestamp_utc": "2026-08-26T19:11:44.014198Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 115364331520, + "reserved_bytes_current_delta": 25096617984, + "reserved_bytes_peak": 115364331520, + "reserved_bytes_peak_delta": 25096617984, + "segment_count_current": 1934, + "segment_count_current_delta": 216, + "segment_count_peak": 1934, + "segment_count_peak_delta": 216, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T191144855658Z.pickle", + "timestamp_utc": "2026-08-26T19:11:44.893584Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191144855658Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191144855658Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a6235a19b59d5c7cf28947095e2d8f605d2bfbe1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191144855658Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:842aaca6286d59cdbbdfe51dbdabd9bf0e10ef6f095f846006e8c4f5002251be +size 1833398 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191145209616Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191145209616Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0c12d5bc741b8ff3f5c13aa0735b12297940b3d1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191145209616Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113344452608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 6, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:45.209192Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113344452608, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 6, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:44.897657Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 115364331520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115364331520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1934, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T191145209616Z.pickle", + "timestamp_utc": "2026-08-26T19:11:45.241309Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191145209616Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191145209616Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a5c58b5764ce3f48d87d00fac759e9c95c5629d5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191145209616Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de4434e63c4efbf4e10d9abb58f7e98e6f5461a4846d73049ac2401d5062f399 +size 1828562 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191147859691Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191147859691Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5370a9dbfb4597ccc3c02cda6c51bf8d0ab760c9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191147859691Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 8193024, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 6, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:47.859146Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 8193024, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 6, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:45.245028Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 108785664, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 1124988416, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 1029701632, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 1029701632, + "segment_count_current": 1701, + "segment_count_current_delta": -233, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T191147859691Z.pickle", + "timestamp_utc": "2026-08-26T19:11:47.898763Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191147859691Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191147859691Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b2ed3a8a2ef0f7f54ba34b1e81d196c41155b0db --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191147859691Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55de74e8a101f445a65e3e01748fb017e3e68ad03ad2952b9bf195efea699a04 +size 2193889 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191148200932Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191148200932Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9c0993e2fd5309e943d017f0b8f718affa117f57 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191148200932Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 6, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.200540Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 6, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:47.902962Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1701, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T191148200932Z.pickle", + "timestamp_utc": "2026-08-26T19:11:48.209430Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191148200932Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191148200932Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..82b1cf3127afc03004cf40c9e2c5da00b7305643 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191148200932Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae332706ae430d7d60b15f19aa03970bd4a91710d5b4549be5c756fcd5dcb751 +size 411608 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191148509693Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191148509693Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a92a4a23daae89ef4bb91f2a2ea5cabd363ae734 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191148509693Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 6, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.509317Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 6, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.210605Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1701, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T191148509693Z.pickle", + "timestamp_utc": "2026-08-26T19:11:48.518139Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191148509693Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191148509693Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2279c474adf0e29213d816782513d4f0059430af --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191148509693Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:985f7b6fd874855158a49c35244f77c7845d091911b6e17250c8c4aa2aa5ef8d +size 411608 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191143969241Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191143969241Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2aef53b033d3053596c752cfe420d146a5c0263f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191143969241Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 32778758656, + "active_bytes_peak": 82480875008, + "active_bytes_peak_delta": 37934908928, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 82480875008, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 82480875008, + "device": 7, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 90267713536, + "reserved_bytes_peak": 90267713536, + "segment_count_current": 1718, + "segment_count_peak": 1718, + "timestamp_utc": "2026-08-26T19:11:43.968727Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 32778758656, + "allocated_bytes_peak": 82480875008, + "allocated_bytes_peak_delta": 37934908928, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 7, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:11:17.937279Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 947639808, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 8523717120, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 90267713536, + "reserved_bytes_current_delta": 42016440320, + "reserved_bytes_peak": 90267713536, + "reserved_bytes_peak_delta": 42016440320, + "segment_count_current": 1718, + "segment_count_current_delta": 1097, + "segment_count_peak": 1718, + "segment_count_peak_delta": 1097, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T191143969241Z.pickle", + "timestamp_utc": "2026-08-26T19:11:44.008394Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191143969241Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191143969241Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..72143f8071abcab41ec4d5d91beb2ef2ce96c43f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191143969241Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2501cc229943c762e85d97c15ae55adbf0f16a5f46a8d74d9e09c1238523858 +size 2304358 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191144855540Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191144855540Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e69e747786b3370b289662e1f77f42ae08b2a15b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191144855540Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113344452608, + "active_bytes_peak_delta": 30863577600, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 7, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:44.855115Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113344452608, + "allocated_bytes_peak_delta": 30863577600, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 82480875008, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 82480875008, + "device": 7, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 90267713536, + "reserved_bytes_peak": 90267713536, + "segment_count_current": 1718, + "segment_count_peak": 1718, + "timestamp_utc": "2026-08-26T19:11:44.013043Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 115364331520, + "reserved_bytes_current_delta": 25096617984, + "reserved_bytes_peak": 115364331520, + "reserved_bytes_peak_delta": 25096617984, + "segment_count_current": 1934, + "segment_count_current_delta": 216, + "segment_count_peak": 1934, + "segment_count_peak_delta": 216, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T191144855540Z.pickle", + "timestamp_utc": "2026-08-26T19:11:44.892971Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191144855540Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191144855540Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..efc7c693b49ae64bb9d56436d499338a80714c1c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191144855540Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc3d5cbf1748dec6d56841fa8c36e2e1ac96d61237c3aed9fcb6dfdd2fd94530 +size 1833399 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191145209506Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191145209506Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f1a8098990df66eebcb8040f092f62a01af06649 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191145209506Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77299558912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113344452608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 7, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:45.209096Z", + "world_size": 8 + }, + "allocated_bytes_current": 77299558912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113344452608, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 7, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:44.897062Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9548864000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 115364331520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115364331520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1934, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T191145209506Z.pickle", + "timestamp_utc": "2026-08-26T19:11:45.240859Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191145209506Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191145209506Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..80e94860682895685cbac6ee615a20927393b6f3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191145209506Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65aaf2f2800195519b8500c37b309c1ea0c63dbea5c18781a9d563c60f0a8116 +size 1828562 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191147859483Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191147859483Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e57b19183ecbe72e68971d20c8c86d84f8583e25 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191147859483Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 8193024, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 7, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:47.859053Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 8193024, + "before": { + "active_bytes_current": 77299558912, + "active_bytes_peak": 113344452608, + "allocated_bytes_current": 77299558912, + "allocated_bytes_peak": 113344452608, + "device": 7, + "inactive_split_bytes_current": 1368806912, + "inactive_split_bytes_peak": 9548864000, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 115364331520, + "reserved_bytes_peak": 115364331520, + "segment_count_current": 1934, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:45.244545Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 108785664, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 1124988416, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 1029701632, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 1029701632, + "segment_count_current": 1701, + "segment_count_current_delta": -233, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T191147859483Z.pickle", + "timestamp_utc": "2026-08-26T19:11:47.897537Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191147859483Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191147859483Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..38bf2aca5778b7f03d1c00ad976d84d15b143c37 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191147859483Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2cd666535e6794081fa8fa5c6157b5acbf1f2899207645f0ae035bc7c820da1 +size 2193889 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191148200891Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191148200891Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4af25f2e59919793ae8a8cdff75f6ce70a1471bd --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191148200891Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 7, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.200494Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 7, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:47.901655Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1701, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T191148200891Z.pickle", + "timestamp_utc": "2026-08-26T19:11:48.209220Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191148200891Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191148200891Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..07a7d73f729fdb996a36baa53151a7af50d2edfe --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191148200891Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdfaca27a5f62ba9d19ab8c73624dfe03425daa05326fee69be4a1151a9f4b60 +size 411608 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191148509591Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191148509591Z.json new file mode 100644 index 0000000000000000000000000000000000000000..91686c8ec0b0687a6ae3d89e143caf35ae73e5e9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191148509591Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77316602368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113352645632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 7, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.509214Z", + "world_size": 8 + }, + "allocated_bytes_current": 77316602368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113352645632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77316602368, + "active_bytes_peak": 113352645632, + "allocated_bytes_current": 77316602368, + "allocated_bytes_peak": 113352645632, + "device": 7, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_peak": 10673852416, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116394033152, + "reserved_bytes_peak": 116394033152, + "segment_count_current": 1701, + "segment_count_peak": 1934, + "timestamp_utc": "2026-08-26T19:11:48.210386Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1477592576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10673852416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116394033152, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116394033152, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1701, + "segment_count_current_delta": 0, + "segment_count_peak": 1934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T191148509591Z.pickle", + "timestamp_utc": "2026-08-26T19:11:48.518128Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191148509591Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191148509591Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..69dec505cdb065b5966246121a8b473954afa469 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191148509591Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d25086333494b5badd7e65183857b98d08cfb1382234c87de0813a93b502c245 +size 411608 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log index a861605ef318ee1bb78666ae49406452f052a4e0..3c2a0a838bb905fbfee6c0220b18271e4c04359a 100644 --- a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log @@ -11,3 +11,1433 @@ W0826 19:10:50.217000 627020 torch/distributed/run.py:766] W0826 19:10:50.217000 627020 torch/distributed/run.py:766] ***************************************** W0826 19:10:50.217000 627020 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. W0826 19:10:50.217000 627020 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 12 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 12288 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1536 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 16 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 12 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 5 is connected to 72 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : +7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 01 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7[Gloo] Rank peer ranks. 6Expected number of connected peer ranks is : is connected to 77 + peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W826 19:11:08.372489192 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:11:08.372557856 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:11:08.378509446 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:11:08.378637296 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:11:08.378661410 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:11:08.379532391 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:11:08.379663086 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:11:08.548451013 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.772 seconds +time to initialize megatron (seconds): 7.647 +[after megatron is initialized] datetime: 2026-08-26 19:11:14 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 8521882624 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 14 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight +Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight +Params for bucket 4 (71835904 elements, 71835904 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.pre_mlp_layernorm.weight +Params for bucket 5 (71835904 elements, 71835904 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight +Params for bucket 6 (71835904 elements, 71835904 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.q_layernorm.weight +Params for bucket 7 (71835904 elements, 71835904 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight +Params for bucket 8 (71835904 elements, 71835904 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 9 (71835904 elements, 71835904 padded size): + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 10 (71835904 elements, 71835904 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.q_layernorm.weight +Params for bucket 11 (71835904 elements, 71835904 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.pre_mlp_layernorm.weight +Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 13 (71835904 elements, 71835904 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 14 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 +Params for bucket 4 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 +Params for bucket 5 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 +Params for bucket 6 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 +Params for bucket 9 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 +Params for bucket 10 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 +Params for bucket 11 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 +Params for bucket 12 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 +Params for bucket 15 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 +Params for bucket 16 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 +Params for bucket 17 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 +Params for bucket 18 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 +Params for bucket 23 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 +Params for bucket 24 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 25 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 +Params for bucket 28 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 29 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 +Params for bucket 30 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 +Params for bucket 31 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 +Params for bucket 34 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 +Params for bucket 35 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 +Params for bucket 36 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 37 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 +Params for bucket 42 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 +Params for bucket 43 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 44 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 +Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 +Params for bucket 48 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 +Params for bucket 49 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 +Params for bucket 50 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 +Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 +Params for bucket 54 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 +Params for bucket 55 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 +Params for bucket 56 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 +Params for bucket 61 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 +Params for bucket 62 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 63 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 66 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 +Params for bucket 67 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 +Params for bucket 68 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 +Params for bucket 69 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 70 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 71 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 +Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 73 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 +Params for bucket 74 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 +Params for bucket 75 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 +Params for bucket 76 (37748736 elements, 37748736 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:11:17 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:11:17 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3537.43, 3545.31) + train/valid/test-data-iterators-setup ..........: (69.12, 87.43) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 19:11:17 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:11:43] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 26031.1 | throughput per GPU (TFLOP/s/GPU): 6.9 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.116034E+01 | loss scale: 1.0 | grad norm: 44.796 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 65536 | min safe capacity: 1.0000 | +Number of parameters in transformer block in billions: 29.85 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 30.26 +Number of parameters in most loaded shard in billions: 30.2590 +Theoretical memory footprints: weight and optimizer=216428.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 73717.48291015625 | max allocated: 78657.05224609375 | reserved: 86070.0 | max reserved: 86070.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 19:11:44] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 886.8 | throughput per GPU (TFLOP/s/GPU): 203.9 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117609E+01 | loss scale: 1.0 | grad norm: 44.347 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 65536 | min safe capacity: 1.0000 | + [2026-08-26 19:11:45] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 354.0 | throughput per GPU (TFLOP/s/GPU): 510.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.687605E+00 | loss scale: 1.0 | grad norm: 63.448 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 65536 | min safe capacity: 1.0000 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 19:11:47] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 2649.9 | throughput per GPU (TFLOP/s/GPU): 68.2 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338355E+01 | loss scale: 1.0 | grad norm: 20.184 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 65536 | min safe capacity: 1.0000 | + [2026-08-26 19:11:48] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 341.5 | throughput per GPU (TFLOP/s/GPU): 529.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.035239E+01 | loss scale: 1.0 | grad norm: 92.760 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 65536 | min safe capacity: 1.0000 | + [2026-08-26 19:11:48] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 308.7 | throughput per GPU (TFLOP/s/GPU): 585.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.162630E+01 | loss scale: 1.0 | grad norm: 99.337 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 65536 | min safe capacity: 1.0000 | +[after training is done] datetime: 2026-08-26 19:11:48 +W0826 19:11:48.889000 627020 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:11:48.890000 627020 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 627111 closing signal SIGTERM +W0826 19:11:48.892000 627020 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 627112 closing signal SIGTERM +W0826 19:11:48.897000 627020 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 627113 closing signal SIGTERM +W0826 19:11:48.899000 627020 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 627114 closing signal SIGTERM +W0826 19:11:48.903000 627020 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 627115 closing signal SIGTERM +W0826 19:11:48.905000 627020 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 627116 closing signal SIGTERM +W0826 19:11:48.909000 627020 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 627117 closing signal SIGTERM +W0826 19:11:48.912000 627020 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 627118 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 627020 got signal: 15 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..8f210fd2ec1c21bc26436a724657e358f4f71d30 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,200 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 2 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 27 +3, 629 +4, 4 +5, 45 +6, 47 +7, 629 +0, 831 +1, 833 +2, 839 +3, 833 +4, 839 +5, 839 +6, 843 +7, 849 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 2497 +1, 2487 +2, 2517 +3, 2493 +4, 2493 +5, 2469 +6, 2475 +7, 2483 +0, 15033 +1, 43469 +2, 22733 +3, 15427 +4, 14457 +5, 14769 +6, 14321 +7, 14321 +0, 52185 +1, 52057 +2, 52057 +3, 52057 +4, 51769 +5, 52057 +6, 52057 +7, 52057 +0, 53767 +1, 53777 +2, 53777 +3, 53807 +4, 53519 +5, 53817 +6, 53817 +7, 53797 +0, 54651 +1, 54651 +2, 54651 +3, 54651 +4, 54363 +5, 54651 +6, 54651 +7, 54651 +0, 55227 +1, 55227 +2, 55227 +3, 55227 +4, 54939 +5, 55227 +6, 55227 +7, 55227 +0, 56951 +1, 56951 +2, 56951 +3, 56951 +4, 56663 +5, 56951 +6, 56951 +7, 56951 +0, 67267 +1, 67319 +2, 67281 +3, 67269 +4, 67267 +5, 67343 +6, 67357 +7, 67343 +0, 89773 +1, 89825 +2, 89825 +3, 89825 +4, 89785 +5, 89825 +6, 89825 +7, 89825 +0, 89773 +1, 89825 +2, 89825 +3, 89825 +4, 89785 +5, 89825 +6, 89825 +7, 89825 +0, 89773 +1, 89825 +2, 89825 +3, 89825 +4, 89785 +5, 89825 +6, 89825 +7, 89825 +0, 90083 +1, 90107 +2, 90169 +3, 90151 +4, 90189 +5, 90161 +6, 90125 +7, 90165 +0, 94393 +1, 94351 +2, 94359 +3, 94357 +4, 94489 +5, 94335 +6, 94327 +7, 94399 +0, 117219 +1, 112491 +2, 110155 +3, 109207 +4, 110855 +5, 117295 +6, 116959 +7, 121427 +0, 90367 +1, 91341 +2, 90311 +3, 92999 +4, 94203 +5, 90361 +6, 91199 +7, 90275 +0, 122159 +1, 122191 +2, 122191 +3, 122193 +4, 122445 +5, 122193 +6, 122193 +7, 122193 +0, 5275 +1, 3678 +2, 3678 +3, 3678 +4, 3678 +5, 3678 +6, 3998 +7, 122191 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191249381140Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191249381140Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9f736dc127caf94352d39beb34344060a5f1e745 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191249381140Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77298399744, + "active_bytes_current_delta": 32777599488, + "active_bytes_peak": 83652089344, + "active_bytes_peak_delta": 39106123264, + "after": { + "active_bytes_current": 77298399744, + "active_bytes_peak": 83652089344, + "allocated_bytes_current": 77298399744, + "allocated_bytes_peak": 83652089344, + "device": 0, + "inactive_split_bytes_current": 1044907520, + "inactive_split_bytes_peak": 12098768384, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 88875204608, + "reserved_bytes_peak": 88875204608, + "segment_count_current": 1548, + "segment_count_peak": 1548, + "timestamp_utc": "2026-08-26T19:12:49.380690Z", + "world_size": 8 + }, + "allocated_bytes_current": 77298399744, + "allocated_bytes_current_delta": 32777599488, + "allocated_bytes_peak": 83652089344, + "allocated_bytes_peak_delta": 39106123264, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 0, + "inactive_split_bytes_current": 446332928, + "inactive_split_bytes_peak": 1050312704, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:12:23.497466Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1044907520, + "inactive_split_bytes_current_delta": 598574592, + "inactive_split_bytes_peak": 12098768384, + "inactive_split_bytes_peak_delta": 11048455680, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 88875204608, + "reserved_bytes_current_delta": 40623931392, + "reserved_bytes_peak": 88875204608, + "reserved_bytes_peak_delta": 40623931392, + "segment_count_current": 1548, + "segment_count_current_delta": 927, + "segment_count_peak": 1548, + "segment_count_peak_delta": 927, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T191249381140Z.pickle", + "timestamp_utc": "2026-08-26T19:12:49.420426Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191249381140Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191249381140Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..74d16b9979fe2f651395067848f97fb87c229f55 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191249381140Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9880235e4e1dcb41f15f4e636484b88df880b210a4d1190a7caa61c0ceaead64 +size 2291148 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191250332340Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191250332340Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6e344e2e4a3430679940f498498868843ef7d423 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191250332340Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77298399744, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114516188160, + "active_bytes_peak_delta": 30864098816, + "after": { + "active_bytes_current": 77298399744, + "active_bytes_peak": 114516188160, + "allocated_bytes_current": 77298399744, + "allocated_bytes_peak": 114516188160, + "device": 0, + "inactive_split_bytes_current": 1044907520, + "inactive_split_bytes_peak": 12098768384, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116557611008, + "reserved_bytes_peak": 116557611008, + "segment_count_current": 1839, + "segment_count_peak": 1839, + "timestamp_utc": "2026-08-26T19:12:50.331912Z", + "world_size": 8 + }, + "allocated_bytes_current": 77298399744, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114516188160, + "allocated_bytes_peak_delta": 30864098816, + "before": { + "active_bytes_current": 77298399744, + "active_bytes_peak": 83652089344, + "allocated_bytes_current": 77298399744, + "allocated_bytes_peak": 83652089344, + "device": 0, + "inactive_split_bytes_current": 1044907520, + "inactive_split_bytes_peak": 12098768384, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 88875204608, + "reserved_bytes_peak": 88875204608, + "segment_count_current": 1548, + "segment_count_peak": 1548, + "timestamp_utc": "2026-08-26T19:12:49.425002Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1044907520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12098768384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116557611008, + "reserved_bytes_current_delta": 27682406400, + "reserved_bytes_peak": 116557611008, + "reserved_bytes_peak_delta": 27682406400, + "segment_count_current": 1839, + "segment_count_current_delta": 291, + "segment_count_peak": 1839, + "segment_count_peak_delta": 291, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T191250332340Z.pickle", + "timestamp_utc": "2026-08-26T19:12:50.368334Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191250332340Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191250332340Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..643ffcfa36e935f677602c7908b61c1e500cea84 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191250332340Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a33be9468693cf7df1aab88abacd2fbd00c184e49114007d722700e39925e75 +size 1836003 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191250686094Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191250686094Z.json new file mode 100644 index 0000000000000000000000000000000000000000..59281e3ff0b301521fecbfc10d2219f85d5ad3a7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191250686094Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77298399744, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114523560960, + "active_bytes_peak_delta": 7372800, + "after": { + "active_bytes_current": 77298399744, + "active_bytes_peak": 114523560960, + "allocated_bytes_current": 77298399744, + "allocated_bytes_peak": 114523560960, + "device": 0, + "inactive_split_bytes_current": 1047004672, + "inactive_split_bytes_peak": 12098768384, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116557611008, + "reserved_bytes_peak": 116557611008, + "segment_count_current": 1839, + "segment_count_peak": 1839, + "timestamp_utc": "2026-08-26T19:12:50.685698Z", + "world_size": 8 + }, + "allocated_bytes_current": 77298399744, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114523560960, + "allocated_bytes_peak_delta": 7372800, + "before": { + "active_bytes_current": 77298399744, + "active_bytes_peak": 114516188160, + "allocated_bytes_current": 77298399744, + "allocated_bytes_peak": 114516188160, + "device": 0, + "inactive_split_bytes_current": 1044907520, + "inactive_split_bytes_peak": 12098768384, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116557611008, + "reserved_bytes_peak": 116557611008, + "segment_count_current": 1839, + "segment_count_peak": 1839, + "timestamp_utc": "2026-08-26T19:12:50.372250Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1047004672, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 12098768384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116557611008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116557611008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1839, + "segment_count_current_delta": 0, + "segment_count_peak": 1839, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T191250686094Z.pickle", + "timestamp_utc": "2026-08-26T19:12:50.716655Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191250686094Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191250686094Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d7dbd2b9d4a3f06d285b094e9c60f92375134646 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191250686094Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:309290ba9410076d70efdd9763854037a1c625bf08cce0befc1215af2c9e81ad +size 1822880 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191253369174Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191253369174Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef44dec92be2649e02409ba7dd7387a40b02e967 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191253369174Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77315443200, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 114524630016, + "active_bytes_peak_delta": 1069056, + "after": { + "active_bytes_current": 77315443200, + "active_bytes_peak": 114524630016, + "allocated_bytes_current": 77315443200, + "allocated_bytes_peak": 114524630016, + "device": 0, + "inactive_split_bytes_current": 1166276096, + "inactive_split_bytes_peak": 12098768384, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117421637632, + "reserved_bytes_peak": 117421637632, + "segment_count_current": 1762, + "segment_count_peak": 1839, + "timestamp_utc": "2026-08-26T19:12:53.368759Z", + "world_size": 8 + }, + "allocated_bytes_current": 77315443200, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 114524630016, + "allocated_bytes_peak_delta": 1069056, + "before": { + "active_bytes_current": 77298399744, + "active_bytes_peak": 114523560960, + "allocated_bytes_current": 77298399744, + "allocated_bytes_peak": 114523560960, + "device": 0, + "inactive_split_bytes_current": 1047004672, + "inactive_split_bytes_peak": 12098768384, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116557611008, + "reserved_bytes_peak": 116557611008, + "segment_count_current": 1839, + "segment_count_peak": 1839, + "timestamp_utc": "2026-08-26T19:12:50.720276Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1166276096, + "inactive_split_bytes_current_delta": 119271424, + "inactive_split_bytes_peak": 12098768384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117421637632, + "reserved_bytes_current_delta": 864026624, + "reserved_bytes_peak": 117421637632, + "reserved_bytes_peak_delta": 864026624, + "segment_count_current": 1762, + "segment_count_current_delta": -77, + "segment_count_peak": 1839, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T191253369174Z.pickle", + "timestamp_utc": "2026-08-26T19:12:53.407024Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191253369174Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191253369174Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b5d64e02874cdb6486950f1a749e90c67015d724 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191253369174Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fffa5fe9b27ff8c280f7050595ef2b11dcd697268e20263a1ccd8a7870cfc4cc +size 2118689 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191253712825Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191253712825Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e69dc7a1e25993b838bf29f34821bba73100e7f9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191253712825Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77315443200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114524630016, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77315443200, + "active_bytes_peak": 114524630016, + "allocated_bytes_current": 77315443200, + "allocated_bytes_peak": 114524630016, + "device": 0, + "inactive_split_bytes_current": 1166276096, + "inactive_split_bytes_peak": 12098768384, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117421637632, + "reserved_bytes_peak": 117421637632, + "segment_count_current": 1762, + "segment_count_peak": 1839, + "timestamp_utc": "2026-08-26T19:12:53.712443Z", + "world_size": 8 + }, + "allocated_bytes_current": 77315443200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114524630016, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77315443200, + "active_bytes_peak": 114524630016, + "allocated_bytes_current": 77315443200, + "allocated_bytes_peak": 114524630016, + "device": 0, + "inactive_split_bytes_current": 1166276096, + "inactive_split_bytes_peak": 12098768384, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117421637632, + "reserved_bytes_peak": 117421637632, + "segment_count_current": 1762, + "segment_count_peak": 1839, + "timestamp_utc": "2026-08-26T19:12:53.411079Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1166276096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12098768384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117421637632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117421637632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1762, + "segment_count_current_delta": 0, + "segment_count_peak": 1839, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T191253712825Z.pickle", + "timestamp_utc": "2026-08-26T19:12:53.722045Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191253712825Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191253712825Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..28bffeb8523461b444610003009e4f900fbd5e9a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191253712825Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e94bb9c56e093defb2638d0b7d8b0580b981cc1ef82365240a8a7e7355b2302 +size 437007 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191254023814Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191254023814Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3fd6b8bad347794e995e4dd3e830782ca1bc4607 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191254023814Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77315443200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114524630016, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77315443200, + "active_bytes_peak": 114524630016, + "allocated_bytes_current": 77315443200, + "allocated_bytes_peak": 114524630016, + "device": 0, + "inactive_split_bytes_current": 1166276096, + "inactive_split_bytes_peak": 12098768384, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117421637632, + "reserved_bytes_peak": 117421637632, + "segment_count_current": 1762, + "segment_count_peak": 1839, + "timestamp_utc": "2026-08-26T19:12:54.023439Z", + "world_size": 8 + }, + "allocated_bytes_current": 77315443200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114524630016, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77315443200, + "active_bytes_peak": 114524630016, + "allocated_bytes_current": 77315443200, + "allocated_bytes_peak": 114524630016, + "device": 0, + "inactive_split_bytes_current": 1166276096, + "inactive_split_bytes_peak": 12098768384, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117421637632, + "reserved_bytes_peak": 117421637632, + "segment_count_current": 1762, + "segment_count_peak": 1839, + "timestamp_utc": "2026-08-26T19:12:53.723169Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1166276096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12098768384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117421637632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117421637632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1762, + "segment_count_current_delta": 0, + "segment_count_peak": 1839, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T191254023814Z.pickle", + "timestamp_utc": "2026-08-26T19:12:54.032889Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191254023814Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191254023814Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0048b102bbd1fa705dba00df2f9900fda43c2293 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191254023814Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1620835b40c0dfd2c3870450cf295627c1a633fc4ecd2a6feb7923887c314ce +size 437007 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191249381129Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191249381129Z.json new file mode 100644 index 0000000000000000000000000000000000000000..60b0ea02c87fc7505d1273fd70895d1e31f7778c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191249381129Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 32780242944, + "active_bytes_peak": 83654339584, + "active_bytes_peak_delta": 39108373504, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 83654339584, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 83654339584, + "device": 1, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 88803901440, + "reserved_bytes_peak": 88803901440, + "segment_count_current": 1548, + "segment_count_peak": 1548, + "timestamp_utc": "2026-08-26T19:12:49.380670Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 32780242944, + "allocated_bytes_peak": 83654339584, + "allocated_bytes_peak_delta": 39108373504, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 1, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:12:23.497455Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 656748544, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 11703811584, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 88803901440, + "reserved_bytes_current_delta": 40552628224, + "reserved_bytes_peak": 88803901440, + "reserved_bytes_peak_delta": 40552628224, + "segment_count_current": 1548, + "segment_count_current_delta": 927, + "segment_count_peak": 1548, + "segment_count_peak_delta": 927, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T191249381129Z.pickle", + "timestamp_utc": "2026-08-26T19:12:49.420302Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191249381129Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191249381129Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aaaed26123910565558250632a14845c3da06d2d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191249381129Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e22a3c3a260562d1d11501b14d5c83fcfb06b4b45654c85708d6b5ef1f60744c +size 2278195 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191250332362Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191250332362Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7a3cc55b10c1979bedcb3ba0f0f8862a258c1fc0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191250332362Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114518798848, + "active_bytes_peak_delta": 30864459264, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114518798848, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114518798848, + "device": 1, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.331931Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114518798848, + "allocated_bytes_peak_delta": 30864459264, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 83654339584, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 83654339584, + "device": 1, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 88803901440, + "reserved_bytes_peak": 88803901440, + "segment_count_current": 1548, + "segment_count_peak": 1548, + "timestamp_utc": "2026-08-26T19:12:49.424919Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116675051520, + "reserved_bytes_current_delta": 27871150080, + "reserved_bytes_peak": 116675051520, + "reserved_bytes_peak_delta": 27871150080, + "segment_count_current": 1852, + "segment_count_current_delta": 304, + "segment_count_peak": 1852, + "segment_count_peak_delta": 304, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T191250332362Z.pickle", + "timestamp_utc": "2026-08-26T19:12:50.369147Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191250332362Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191250332362Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eadd24b2ec8d575522f87c1b3e05808574802fb0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191250332362Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59aab60fb162b41d4f63e3709fb4851f127a6131aadd0d3471c46b40656d54a8 +size 1826803 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191250686121Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191250686121Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3ca112259b941f8e972d05981c77a9a2f187038d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191250686121Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114526138880, + "active_bytes_peak_delta": 7340032, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114526138880, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114526138880, + "device": 1, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.685723Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114526138880, + "allocated_bytes_peak_delta": 7340032, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114518798848, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114518798848, + "device": 1, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.373091Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116675051520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116675051520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1852, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T191250686121Z.pickle", + "timestamp_utc": "2026-08-26T19:12:50.716941Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191250686121Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191250686121Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..84691333585b61cf3b3b14af6c49a2cbe12b040f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191250686121Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:005fafdb001f0cb69f239c3b248dd07698a1b112fc8ee16658e194899834d9f3 +size 1815734 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191253369158Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191253369158Z.json new file mode 100644 index 0000000000000000000000000000000000000000..096a0d0549ecea51fcbcae4cb3a423bedee8a117 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191253369158Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 1134592, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 1, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.368731Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 1134592, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114526138880, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114526138880, + "device": 1, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.720548Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 29093888, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 780140544, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 780140544, + "segment_count_current": 1763, + "segment_count_current_delta": -89, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T191253369158Z.pickle", + "timestamp_utc": "2026-08-26T19:12:53.407669Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191253369158Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191253369158Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..41c63587be2be2e64bb0493f472c7ebc85d7eaac --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191253369158Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0ff52f96f8d1ba8297b5c48a944f9f2ffa94b33705d5968b82e673e3ced9ae4 +size 2123239 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191253712833Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191253712833Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5c376a010e0dca83184ac4809635d3cc3aa2e836 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191253712833Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 1, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.712445Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 1, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.411783Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1763, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T191253712833Z.pickle", + "timestamp_utc": "2026-08-26T19:12:53.721929Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191253712833Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191253712833Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..884eb2ad9d621f0de634532242f4ab471d0156c8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191253712833Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33f123e0ed2e7904651ca86959339d85f3e1feb701939eab1204f67d8400a613 +size 427803 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191254023879Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191254023879Z.json new file mode 100644 index 0000000000000000000000000000000000000000..52b1d5904cbc2df5c1a650edb5e5fe10cf9eb602 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191254023879Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 1, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:54.023502Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 1, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.723127Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1763, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T191254023879Z.pickle", + "timestamp_utc": "2026-08-26T19:12:54.032774Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191254023879Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191254023879Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c79f88ea03f1ca6a1a21e6c42e3fadf3565d40f5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191254023879Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e97f4ee0370a799d060c4380752b0bf3fe34e7726e67d2c438993f972c5ad67 +size 427803 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191249381129Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191249381129Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5acbed4f9e24c24cef4587a1d49a089eeae846ea --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191249381129Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 32780242944, + "active_bytes_peak": 83654339584, + "active_bytes_peak_delta": 39108373504, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 83654339584, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 83654339584, + "device": 2, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 88803901440, + "reserved_bytes_peak": 88803901440, + "segment_count_current": 1548, + "segment_count_peak": 1548, + "timestamp_utc": "2026-08-26T19:12:49.380660Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 32780242944, + "allocated_bytes_peak": 83654339584, + "allocated_bytes_peak_delta": 39108373504, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 2, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:12:23.497498Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 656748544, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 11703811584, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 88803901440, + "reserved_bytes_current_delta": 40552628224, + "reserved_bytes_peak": 88803901440, + "reserved_bytes_peak_delta": 40552628224, + "segment_count_current": 1548, + "segment_count_current_delta": 927, + "segment_count_peak": 1548, + "segment_count_peak_delta": 927, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T191249381129Z.pickle", + "timestamp_utc": "2026-08-26T19:12:49.421294Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191249381129Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191249381129Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..94f290b8efe69f2ce425d5228ce559403ed4f5b5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191249381129Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab68f636fffdb689b244575ef7899903bac9292dca6f9ed2a442082f4f287b32 +size 2278195 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191250332317Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191250332317Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8f27fb95a712972bfdb67b7a215d0095999613e8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191250332317Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114518798848, + "active_bytes_peak_delta": 30864459264, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114518798848, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114518798848, + "device": 2, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.331886Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114518798848, + "allocated_bytes_peak_delta": 30864459264, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 83654339584, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 83654339584, + "device": 2, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 88803901440, + "reserved_bytes_peak": 88803901440, + "segment_count_current": 1548, + "segment_count_peak": 1548, + "timestamp_utc": "2026-08-26T19:12:49.425383Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116675051520, + "reserved_bytes_current_delta": 27871150080, + "reserved_bytes_peak": 116675051520, + "reserved_bytes_peak_delta": 27871150080, + "segment_count_current": 1852, + "segment_count_current_delta": 304, + "segment_count_peak": 1852, + "segment_count_peak_delta": 304, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T191250332317Z.pickle", + "timestamp_utc": "2026-08-26T19:12:50.369893Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191250332317Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191250332317Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e09a5b7d5ead385d0641dc076b4878cb014dd946 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191250332317Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06786b001c192d5a417509359804fce84d273f512bbc572d243503ab2653ce0e +size 1826803 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191250686177Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191250686177Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b3204daf821bcbb7309db0fc770f250f1b868b2b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191250686177Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114526138880, + "active_bytes_peak_delta": 7340032, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114526138880, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114526138880, + "device": 2, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.685765Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114526138880, + "allocated_bytes_peak_delta": 7340032, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114518798848, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114518798848, + "device": 2, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.373838Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116675051520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116675051520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1852, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T191250686177Z.pickle", + "timestamp_utc": "2026-08-26T19:12:50.716470Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191250686177Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191250686177Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dab8690522e7d777061e7baf1f6a863b462a9ee2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191250686177Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea02ae6433d28e93ed966a8a382f5fa07da26f004905fac7fbe8168b245a0479 +size 1815732 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191253369245Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191253369245Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8ac6973977cbc27fe467a129fc27b8060b5af425 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191253369245Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 1134592, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 2, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.368739Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 1134592, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114526138880, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114526138880, + "device": 2, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.720023Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 29093888, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 780140544, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 780140544, + "segment_count_current": 1763, + "segment_count_current_delta": -89, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T191253369245Z.pickle", + "timestamp_utc": "2026-08-26T19:12:53.407299Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191253369245Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191253369245Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d066fd604443a679427bfb4903525259e7bda36a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191253369245Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0772b8768ecaab0de617776f25b71602bf53e4464b985f0ad81fce8ae4ead498 +size 2123237 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191253712825Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191253712825Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aee37833f07f91fd926adcb0fc19b034fc4509d4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191253712825Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 2, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.712435Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 2, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.411223Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1763, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T191253712825Z.pickle", + "timestamp_utc": "2026-08-26T19:12:53.721792Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191253712825Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191253712825Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9f6af236f3b304c681319975ba78def1e02f843e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191253712825Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d157c97c89b55a38f235a181544aba44ac93db3126411052ba1ae9ca93e99817 +size 427801 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191254023838Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191254023838Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3548e2141f0ec11e5e2972409f426c516ae77e07 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191254023838Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 2, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:54.023461Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 2, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.722921Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1763, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T191254023838Z.pickle", + "timestamp_utc": "2026-08-26T19:12:54.032580Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191254023838Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191254023838Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a5a15fab97c1a1865c9e0d4acc460daf74b9d408 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191254023838Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:717a97e06595efdb176cb163629c749ca6ad5bc90f8abf749ec4aaede02b25ec +size 427801 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191249381197Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191249381197Z.json new file mode 100644 index 0000000000000000000000000000000000000000..afe824b761fe7287b8089cb2bf9d52ce9d39ae18 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191249381197Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 32780242944, + "active_bytes_peak": 83654339584, + "active_bytes_peak_delta": 39108373504, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 83654339584, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 83654339584, + "device": 3, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 88803901440, + "reserved_bytes_peak": 88803901440, + "segment_count_current": 1548, + "segment_count_peak": 1548, + "timestamp_utc": "2026-08-26T19:12:49.380723Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 32780242944, + "allocated_bytes_peak": 83654339584, + "allocated_bytes_peak_delta": 39108373504, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 3, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:12:23.497628Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 656748544, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 11703811584, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 88803901440, + "reserved_bytes_current_delta": 40552628224, + "reserved_bytes_peak": 88803901440, + "reserved_bytes_peak_delta": 40552628224, + "segment_count_current": 1548, + "segment_count_current_delta": 927, + "segment_count_peak": 1548, + "segment_count_peak_delta": 927, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T191249381197Z.pickle", + "timestamp_utc": "2026-08-26T19:12:49.420161Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191249381197Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191249381197Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9e792298a936e84916ca6d3b16feec9e35770525 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191249381197Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fda6bddba0bcc155305b028ac4688a5667dd79e7cb4bef8180f33af7ebd7674f +size 2278195 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191250332447Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191250332447Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dd7910c4b1c7d790c20b9681bd0a18532579fd6c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191250332447Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114518798848, + "active_bytes_peak_delta": 30864459264, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114518798848, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114518798848, + "device": 3, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.332006Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114518798848, + "allocated_bytes_peak_delta": 30864459264, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 83654339584, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 83654339584, + "device": 3, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 88803901440, + "reserved_bytes_peak": 88803901440, + "segment_count_current": 1548, + "segment_count_peak": 1548, + "timestamp_utc": "2026-08-26T19:12:49.424832Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116675051520, + "reserved_bytes_current_delta": 27871150080, + "reserved_bytes_peak": 116675051520, + "reserved_bytes_peak_delta": 27871150080, + "segment_count_current": 1852, + "segment_count_current_delta": 304, + "segment_count_peak": 1852, + "segment_count_peak_delta": 304, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T191250332447Z.pickle", + "timestamp_utc": "2026-08-26T19:12:50.369342Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191250332447Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191250332447Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..394dc086a96ca88fc293e166db866ac8c4cd8d6a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191250332447Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f85a8086de42ff094dfff38a1496bf35d1d4a47cdae993ac1eb6d82656845fda +size 1826800 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191250686220Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191250686220Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dfbda739522dc518be203868839b764f53070b55 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191250686220Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114526138880, + "active_bytes_peak_delta": 7340032, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114526138880, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114526138880, + "device": 3, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.685798Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114526138880, + "allocated_bytes_peak_delta": 7340032, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114518798848, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114518798848, + "device": 3, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.373340Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116675051520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116675051520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1852, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T191250686220Z.pickle", + "timestamp_utc": "2026-08-26T19:12:50.716942Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191250686220Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191250686220Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7216080810192b97e68601e188ec8389335fac58 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191250686220Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49c962ec67dcb2d67e7aafbad71c312e969b3c3e3f1d1f581a46ed53fb9f2af9 +size 1815734 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191253369329Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191253369329Z.json new file mode 100644 index 0000000000000000000000000000000000000000..69f50c0b183f3d2fe99b2360a92da34f5cd815db --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191253369329Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 1134592, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 3, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.368858Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 1134592, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114526138880, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114526138880, + "device": 3, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.720691Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 29093888, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 780140544, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 780140544, + "segment_count_current": 1763, + "segment_count_current_delta": -89, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T191253369329Z.pickle", + "timestamp_utc": "2026-08-26T19:12:53.407081Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191253369329Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191253369329Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9a08b11b00c7667617a3ecdbeded43e162d27304 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191253369329Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c0b895e87d19949e688d2804268d1204073a71aa9f914a267121c86714d6a6f +size 2123237 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191253712891Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191253712891Z.json new file mode 100644 index 0000000000000000000000000000000000000000..47157f265bb505607854f6d7bc57c75cf0825b56 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191253712891Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 3, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.712502Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 3, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.411249Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1763, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T191253712891Z.pickle", + "timestamp_utc": "2026-08-26T19:12:53.721865Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191253712891Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191253712891Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..612215ad31c558c7512a1f60503f49d4cbeeb675 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191253712891Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5281ed1d75434be3b8d99111ac6e6020ee83785ee23393f0a880df60f0a4ae +size 427801 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191254023885Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191254023885Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9bf2ca5caefe1908c8cff146a24c42dc461dc8f8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191254023885Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 3, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:54.023509Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 3, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.723026Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1763, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T191254023885Z.pickle", + "timestamp_utc": "2026-08-26T19:12:54.032724Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191254023885Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191254023885Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a45b8db65fb583cc24b7dae5d7b80ff3774de61f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191254023885Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91c67f74ca6838dab4ae425db84b897f79be0e9874c2cae964450036deeb6ed1 +size 427801 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191249381318Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191249381318Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9a6ab58a6d21066fe7f7bed4bc11ef5fc9c713b3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191249381318Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301979648, + "active_bytes_current_delta": 32781179392, + "active_bytes_peak": 83664184832, + "active_bytes_peak_delta": 39118218752, + "after": { + "active_bytes_current": 77301979648, + "active_bytes_peak": 83664184832, + "allocated_bytes_current": 77301979648, + "allocated_bytes_peak": 83664184832, + "device": 4, + "inactive_split_bytes_current": 1343317504, + "inactive_split_bytes_peak": 11890516992, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 88963284992, + "reserved_bytes_peak": 88963284992, + "segment_count_current": 1554, + "segment_count_peak": 1554, + "timestamp_utc": "2026-08-26T19:12:49.380806Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301979648, + "allocated_bytes_current_delta": 32781179392, + "allocated_bytes_peak": 83664184832, + "allocated_bytes_peak_delta": 39118218752, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 4, + "inactive_split_bytes_current": 270172160, + "inactive_split_bytes_peak": 874151936, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 47949283328, + "reserved_bytes_peak": 47949283328, + "segment_count_current": 615, + "segment_count_peak": 615, + "timestamp_utc": "2026-08-26T19:12:23.497550Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1343317504, + "inactive_split_bytes_current_delta": 1073145344, + "inactive_split_bytes_peak": 11890516992, + "inactive_split_bytes_peak_delta": 11016365056, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 88963284992, + "reserved_bytes_current_delta": 41014001664, + "reserved_bytes_peak": 88963284992, + "reserved_bytes_peak_delta": 41014001664, + "segment_count_current": 1554, + "segment_count_current_delta": 939, + "segment_count_peak": 1554, + "segment_count_peak_delta": 939, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T191249381318Z.pickle", + "timestamp_utc": "2026-08-26T19:12:49.421665Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191249381318Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191249381318Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cff4fe2a935cde086620ab4ffa3ac3c2b3500f16 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191249381318Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d215d5fa9582c9bf873e59f8b911e9e3ad2e5b0fe82130e8375f21a1311f5d66 +size 2286265 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191250332503Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191250332503Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1fe2317c56989aa72e0a77a25796c10df4c18f6a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191250332503Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301979648, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114534276096, + "active_bytes_peak_delta": 30870091264, + "after": { + "active_bytes_current": 77301979648, + "active_bytes_peak": 114534276096, + "allocated_bytes_current": 77301979648, + "allocated_bytes_peak": 114534276096, + "device": 4, + "inactive_split_bytes_current": 1341220352, + "inactive_split_bytes_peak": 11890516992, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116658274304, + "reserved_bytes_peak": 116658274304, + "segment_count_current": 1846, + "segment_count_peak": 1846, + "timestamp_utc": "2026-08-26T19:12:50.332052Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301979648, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114534276096, + "allocated_bytes_peak_delta": 30870091264, + "before": { + "active_bytes_current": 77301979648, + "active_bytes_peak": 83664184832, + "allocated_bytes_current": 77301979648, + "allocated_bytes_peak": 83664184832, + "device": 4, + "inactive_split_bytes_current": 1343317504, + "inactive_split_bytes_peak": 11890516992, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 88963284992, + "reserved_bytes_peak": 88963284992, + "segment_count_current": 1554, + "segment_count_peak": 1554, + "timestamp_utc": "2026-08-26T19:12:49.426391Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1341220352, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 11890516992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116658274304, + "reserved_bytes_current_delta": 27694989312, + "reserved_bytes_peak": 116658274304, + "reserved_bytes_peak_delta": 27694989312, + "segment_count_current": 1846, + "segment_count_current_delta": 292, + "segment_count_peak": 1846, + "segment_count_peak_delta": 292, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T191250332503Z.pickle", + "timestamp_utc": "2026-08-26T19:12:50.369549Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191250332503Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191250332503Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6d099cd07c1c4ea994d0cf2d5fbcae8e98fcb515 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191250332503Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bac163ea36f57d88f3caf8252aff3fedb5e3fe7f9fa5e32c484a3b2b9c34872 +size 1830064 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191250686199Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191250686199Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b1e61de688682cf591303e19c63f14a85ce177a7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191250686199Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301979648, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114541862400, + "active_bytes_peak_delta": 7586304, + "after": { + "active_bytes_current": 77301979648, + "active_bytes_peak": 114541862400, + "allocated_bytes_current": 77301979648, + "allocated_bytes_peak": 114541862400, + "device": 4, + "inactive_split_bytes_current": 1341220352, + "inactive_split_bytes_peak": 11890516992, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116658274304, + "reserved_bytes_peak": 116658274304, + "segment_count_current": 1846, + "segment_count_peak": 1846, + "timestamp_utc": "2026-08-26T19:12:50.685791Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301979648, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114541862400, + "allocated_bytes_peak_delta": 7586304, + "before": { + "active_bytes_current": 77301979648, + "active_bytes_peak": 114534276096, + "allocated_bytes_current": 77301979648, + "allocated_bytes_peak": 114534276096, + "device": 4, + "inactive_split_bytes_current": 1341220352, + "inactive_split_bytes_peak": 11890516992, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116658274304, + "reserved_bytes_peak": 116658274304, + "segment_count_current": 1846, + "segment_count_peak": 1846, + "timestamp_utc": "2026-08-26T19:12:50.373590Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1341220352, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11890516992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116658274304, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116658274304, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1846, + "segment_count_current_delta": 0, + "segment_count_peak": 1846, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T191250686199Z.pickle", + "timestamp_utc": "2026-08-26T19:12:50.717687Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191250686199Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191250686199Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1ad78037762870d0e4c39857595585c2adf420c0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191250686199Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a05a87f4222a3cfa9e5dfd3e4448f5556f608d065b3095ff98e4cce50d0dbe0e +size 1816723 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191253369364Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191253369364Z.json new file mode 100644 index 0000000000000000000000000000000000000000..799fb87c8091c0f8e0552eed61f232a7c732fb07 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191253369364Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77319023104, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 114541862400, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77319023104, + "active_bytes_peak": 114541862400, + "allocated_bytes_current": 77319023104, + "allocated_bytes_peak": 114541862400, + "device": 4, + "inactive_split_bytes_current": 1370314240, + "inactive_split_bytes_peak": 11890516992, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117719433216, + "reserved_bytes_peak": 117719433216, + "segment_count_current": 1787, + "segment_count_peak": 1846, + "timestamp_utc": "2026-08-26T19:12:53.368876Z", + "world_size": 8 + }, + "allocated_bytes_current": 77319023104, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 114541862400, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77301979648, + "active_bytes_peak": 114541862400, + "allocated_bytes_current": 77301979648, + "allocated_bytes_peak": 114541862400, + "device": 4, + "inactive_split_bytes_current": 1341220352, + "inactive_split_bytes_peak": 11890516992, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116658274304, + "reserved_bytes_peak": 116658274304, + "segment_count_current": 1846, + "segment_count_peak": 1846, + "timestamp_utc": "2026-08-26T19:12:50.721413Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1370314240, + "inactive_split_bytes_current_delta": 29093888, + "inactive_split_bytes_peak": 11890516992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117719433216, + "reserved_bytes_current_delta": 1061158912, + "reserved_bytes_peak": 117719433216, + "reserved_bytes_peak_delta": 1061158912, + "segment_count_current": 1787, + "segment_count_current_delta": -59, + "segment_count_peak": 1846, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T191253369364Z.pickle", + "timestamp_utc": "2026-08-26T19:12:53.407738Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191253369364Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191253369364Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..be6068af081b030398c159868b6c5f636b585be0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191253369364Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dad422260aac1320ded84ad02b27845d07a17a4bd163ba16667f0c2f5b153490 +size 2113033 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191253712907Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191253712907Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6366f38925bf7984b8321aa4583d68391501faaf --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191253712907Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77319023104, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114541862400, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77319023104, + "active_bytes_peak": 114541862400, + "allocated_bytes_current": 77319023104, + "allocated_bytes_peak": 114541862400, + "device": 4, + "inactive_split_bytes_current": 1370314240, + "inactive_split_bytes_peak": 11890516992, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117719433216, + "reserved_bytes_peak": 117719433216, + "segment_count_current": 1787, + "segment_count_peak": 1846, + "timestamp_utc": "2026-08-26T19:12:53.712520Z", + "world_size": 8 + }, + "allocated_bytes_current": 77319023104, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114541862400, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77319023104, + "active_bytes_peak": 114541862400, + "allocated_bytes_current": 77319023104, + "allocated_bytes_peak": 114541862400, + "device": 4, + "inactive_split_bytes_current": 1370314240, + "inactive_split_bytes_peak": 11890516992, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117719433216, + "reserved_bytes_peak": 117719433216, + "segment_count_current": 1787, + "segment_count_peak": 1846, + "timestamp_utc": "2026-08-26T19:12:53.411858Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1370314240, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11890516992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117719433216, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117719433216, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1787, + "segment_count_current_delta": 0, + "segment_count_peak": 1846, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T191253712907Z.pickle", + "timestamp_utc": "2026-08-26T19:12:53.722113Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191253712907Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191253712907Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1d07cc728d60fee05093e0ad5dcaa5f86c754ca0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191253712907Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:580dda0471fbe2485ebe9755ac60688678730f4dcb8da65b8f7bb9e92acf0346 +size 434208 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191254023892Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191254023892Z.json new file mode 100644 index 0000000000000000000000000000000000000000..785e8ced9fd901030ff69bcfa250efd879f7fff9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191254023892Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77319023104, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114541862400, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77319023104, + "active_bytes_peak": 114541862400, + "allocated_bytes_current": 77319023104, + "allocated_bytes_peak": 114541862400, + "device": 4, + "inactive_split_bytes_current": 1370314240, + "inactive_split_bytes_peak": 11890516992, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117719433216, + "reserved_bytes_peak": 117719433216, + "segment_count_current": 1787, + "segment_count_peak": 1846, + "timestamp_utc": "2026-08-26T19:12:54.023522Z", + "world_size": 8 + }, + "allocated_bytes_current": 77319023104, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114541862400, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77319023104, + "active_bytes_peak": 114541862400, + "allocated_bytes_current": 77319023104, + "allocated_bytes_peak": 114541862400, + "device": 4, + "inactive_split_bytes_current": 1370314240, + "inactive_split_bytes_peak": 11890516992, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117719433216, + "reserved_bytes_peak": 117719433216, + "segment_count_current": 1787, + "segment_count_peak": 1846, + "timestamp_utc": "2026-08-26T19:12:53.723232Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1370314240, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11890516992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117719433216, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117719433216, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1787, + "segment_count_current_delta": 0, + "segment_count_peak": 1846, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T191254023892Z.pickle", + "timestamp_utc": "2026-08-26T19:12:54.032916Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191254023892Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191254023892Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1e795c51e8756db97493763307cecbee22ac24c2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191254023892Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:deb1847c55503797e31e0c90f8bc90d3f6ead57e5c589b4da68e852b92ad943a +size 434208 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191249381127Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191249381127Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f89494b9c16bed228f21cceb7c2816996740c4c4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191249381127Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 32780242944, + "active_bytes_peak": 83654339584, + "active_bytes_peak_delta": 39108373504, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 83654339584, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 83654339584, + "device": 5, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 88803901440, + "reserved_bytes_peak": 88803901440, + "segment_count_current": 1548, + "segment_count_peak": 1548, + "timestamp_utc": "2026-08-26T19:12:49.380626Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 32780242944, + "allocated_bytes_peak": 83654339584, + "allocated_bytes_peak_delta": 39108373504, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 5, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:12:23.497378Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 656748544, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 11703811584, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 88803901440, + "reserved_bytes_current_delta": 40552628224, + "reserved_bytes_peak": 88803901440, + "reserved_bytes_peak_delta": 40552628224, + "segment_count_current": 1548, + "segment_count_current_delta": 927, + "segment_count_peak": 1548, + "segment_count_peak_delta": 927, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T191249381127Z.pickle", + "timestamp_utc": "2026-08-26T19:12:49.420224Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191249381127Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191249381127Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fde24eb1f1fdf3dd9f5de5f2ef024bb3eb837459 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191249381127Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c851c67fb358b55238631502946013c9f22055f1340907b6aed92f4eef40cc91 +size 2278195 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191250332325Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191250332325Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d7b2d15900cc45a77f2df54d34f2ecf3f166a48 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191250332325Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114518798848, + "active_bytes_peak_delta": 30864459264, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114518798848, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114518798848, + "device": 5, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.331900Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114518798848, + "allocated_bytes_peak_delta": 30864459264, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 83654339584, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 83654339584, + "device": 5, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 88803901440, + "reserved_bytes_peak": 88803901440, + "segment_count_current": 1548, + "segment_count_peak": 1548, + "timestamp_utc": "2026-08-26T19:12:49.424780Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116675051520, + "reserved_bytes_current_delta": 27871150080, + "reserved_bytes_peak": 116675051520, + "reserved_bytes_peak_delta": 27871150080, + "segment_count_current": 1852, + "segment_count_current_delta": 304, + "segment_count_peak": 1852, + "segment_count_peak_delta": 304, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T191250332325Z.pickle", + "timestamp_utc": "2026-08-26T19:12:50.369128Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191250332325Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191250332325Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6b4ae9d9766f76a544cc351463787452bbcdd549 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191250332325Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd07b401dbd912858b272795e73565cf5146fb9eb94ae42f1ae22117cbf7eee5 +size 1826803 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191250686091Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191250686091Z.json new file mode 100644 index 0000000000000000000000000000000000000000..933d5dd493a7736334196800eb94b9d5c995d130 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191250686091Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114526138880, + "active_bytes_peak_delta": 7340032, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114526138880, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114526138880, + "device": 5, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.685693Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114526138880, + "allocated_bytes_peak_delta": 7340032, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114518798848, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114518798848, + "device": 5, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.373132Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116675051520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116675051520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1852, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T191250686091Z.pickle", + "timestamp_utc": "2026-08-26T19:12:50.716666Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191250686091Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191250686091Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..33586ec103715b97e1be9c2b6de1ff46b5a886fd --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191250686091Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7aca4a7435471f7892d5ea50749830b665dd87f0cca1bda87679711f7fb6dbb2 +size 1815732 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191253369244Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191253369244Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e72fd8415d7adb5741724eda1f5f63b0241ef763 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191253369244Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 1134592, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 5, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.368737Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 1134592, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114526138880, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114526138880, + "device": 5, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.720300Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 29093888, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 780140544, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 780140544, + "segment_count_current": 1763, + "segment_count_current_delta": -89, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T191253369244Z.pickle", + "timestamp_utc": "2026-08-26T19:12:53.407239Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191253369244Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191253369244Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9a7366090d5dc13ff4f3a61846b83c5434c005f4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191253369244Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b15c48f6d1c43d81efbd82dbd2a98a01d0d41291af5ba534fb52e40c9eb4835 +size 2123237 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191253712811Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191253712811Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f213e5aa55d14f5cc31b30f5ddee987b52fec5f2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191253712811Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 5, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.712433Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 5, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.411341Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1763, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T191253712811Z.pickle", + "timestamp_utc": "2026-08-26T19:12:53.721728Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191253712811Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191253712811Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9836eeb4dfd807d141adb97bd3213466e7cf364a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191253712811Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e51c76490ddd4e8f3ac1d150eee7ffb9a0ccfcf1cfee6fb72e00a37de32a59d3 +size 427801 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191254023789Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191254023789Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6f036eaf5ea3015049cb30dcb817195eb338e17c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191254023789Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 5, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:54.023418Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 5, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.722889Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1763, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T191254023789Z.pickle", + "timestamp_utc": "2026-08-26T19:12:54.032434Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191254023789Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191254023789Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b5dbb72bc09aaab199e15c0bee26d2961b812b49 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191254023789Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b71cfa07d405b2faf657753ac1f774b3220c5996d0e85a4d668a0eed222756ca +size 427801 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191249381129Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191249381129Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a3929e7fab8ef7698c46b466c604454597405cfb --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191249381129Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 32780242944, + "active_bytes_peak": 83654339584, + "active_bytes_peak_delta": 39108373504, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 83654339584, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 83654339584, + "device": 6, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 88803901440, + "reserved_bytes_peak": 88803901440, + "segment_count_current": 1548, + "segment_count_peak": 1548, + "timestamp_utc": "2026-08-26T19:12:49.380618Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 32780242944, + "allocated_bytes_peak": 83654339584, + "allocated_bytes_peak_delta": 39108373504, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 6, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:12:23.497454Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 656748544, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 11703811584, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 88803901440, + "reserved_bytes_current_delta": 40552628224, + "reserved_bytes_peak": 88803901440, + "reserved_bytes_peak_delta": 40552628224, + "segment_count_current": 1548, + "segment_count_current_delta": 927, + "segment_count_peak": 1548, + "segment_count_peak_delta": 927, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T191249381129Z.pickle", + "timestamp_utc": "2026-08-26T19:12:49.420016Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191249381129Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191249381129Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ac41c5774ea976d10d71a59f65ea309323abceb3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191249381129Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92a8559125b29d75da572e8780367912d319352fb496df023594ff135ba8850f +size 2278195 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191250332325Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191250332325Z.json new file mode 100644 index 0000000000000000000000000000000000000000..704513fd55bd65654bd44c77208aae1c7007e57f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191250332325Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114518798848, + "active_bytes_peak_delta": 30864459264, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114518798848, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114518798848, + "device": 6, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.331889Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114518798848, + "allocated_bytes_peak_delta": 30864459264, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 83654339584, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 83654339584, + "device": 6, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 88803901440, + "reserved_bytes_peak": 88803901440, + "segment_count_current": 1548, + "segment_count_peak": 1548, + "timestamp_utc": "2026-08-26T19:12:49.424567Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116675051520, + "reserved_bytes_current_delta": 27871150080, + "reserved_bytes_peak": 116675051520, + "reserved_bytes_peak_delta": 27871150080, + "segment_count_current": 1852, + "segment_count_current_delta": 304, + "segment_count_peak": 1852, + "segment_count_peak_delta": 304, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T191250332325Z.pickle", + "timestamp_utc": "2026-08-26T19:12:50.369140Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191250332325Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191250332325Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c741c61f8743ff205c71ba7b89e0a030094103fe --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191250332325Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f500895a19958dcac1c66538be555dbbdf6fed4566d83119185c59c1b6139ae8 +size 1826803 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191250686106Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191250686106Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8c9bf23aa2bf7326dc8710e25467ed199048aad1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191250686106Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114526138880, + "active_bytes_peak_delta": 7340032, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114526138880, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114526138880, + "device": 6, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.685706Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114526138880, + "allocated_bytes_peak_delta": 7340032, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114518798848, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114518798848, + "device": 6, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.373030Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116675051520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116675051520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1852, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T191250686106Z.pickle", + "timestamp_utc": "2026-08-26T19:12:50.716104Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191250686106Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191250686106Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bccb4c0cd2d8e3a078d4866b9fa63f91b827adcb --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191250686106Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ef5eef47269d01de61e5218026b14c54401eb836ede330979e7c579e30f93b9 +size 1815732 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191253369170Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191253369170Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3974a205b6dbabe389e9e7fc99cc4d0f533116f0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191253369170Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 1134592, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 6, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.368739Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 1134592, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114526138880, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114526138880, + "device": 6, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.719739Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 29093888, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 780140544, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 780140544, + "segment_count_current": 1763, + "segment_count_current_delta": -89, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T191253369170Z.pickle", + "timestamp_utc": "2026-08-26T19:12:53.407590Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191253369170Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191253369170Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b963d6795fa2da18d85700b2764fce8c9a294afa --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191253369170Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab5e23582167cc1ac9d676fc92ef6a6e0b28546a77c9ee8cdb451b8179cfd36f +size 2123237 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191253712786Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191253712786Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e7edfbe63a93f94c2d2ca07d0ef22c377c446640 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191253712786Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 6, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.712396Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 6, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.411764Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1763, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T191253712786Z.pickle", + "timestamp_utc": "2026-08-26T19:12:53.721985Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191253712786Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191253712786Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..09f3f8d2973d12b30162bc20aa31d77012c9d5d5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191253712786Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6167381d3bf7d268b3b755bf11d49ee99987d9f67c6f0103286fd23e3b824d56 +size 427801 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191254023815Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191254023815Z.json new file mode 100644 index 0000000000000000000000000000000000000000..18b43a2cfd2eb52fd7e39b4b426cf77c9b71511b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191254023815Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 6, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:54.023439Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 6, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.723091Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1763, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T191254023815Z.pickle", + "timestamp_utc": "2026-08-26T19:12:54.032772Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191254023815Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191254023815Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..72b8d40ea4eb2fc9f388b337f6ff3c84d76ed596 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191254023815Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:935e36a49bb0cf953cd23bfdd8faff904ea65f7c1c3b689819e6d9aeef637d9c +size 427801 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191249381113Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191249381113Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e924c9fe05de790c5307e2c99dee56f6964263dc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191249381113Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 32780242944, + "active_bytes_peak": 83654339584, + "active_bytes_peak_delta": 39108373504, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 83654339584, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 83654339584, + "device": 7, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 88803901440, + "reserved_bytes_peak": 88803901440, + "segment_count_current": 1548, + "segment_count_peak": 1548, + "timestamp_utc": "2026-08-26T19:12:49.380608Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 32780242944, + "allocated_bytes_peak": 83654339584, + "allocated_bytes_peak_delta": 39108373504, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 7, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:12:23.497426Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 656748544, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 11703811584, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 88803901440, + "reserved_bytes_current_delta": 40552628224, + "reserved_bytes_peak": 88803901440, + "reserved_bytes_peak_delta": 40552628224, + "segment_count_current": 1548, + "segment_count_current_delta": 927, + "segment_count_peak": 1548, + "segment_count_peak_delta": 927, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T191249381113Z.pickle", + "timestamp_utc": "2026-08-26T19:12:49.419564Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191249381113Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191249381113Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a7f7e5ec318192cf481086158a022a9672415171 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191249381113Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddabfa58e0e4019cd25d4ccb5666031366a3378e819f6af0e268fde8a9e7b15f +size 2278195 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191250332363Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191250332363Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc158d8d737a9d38d1329aa4df2962f424c05ecc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191250332363Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114518798848, + "active_bytes_peak_delta": 30864459264, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114518798848, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114518798848, + "device": 7, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.331937Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114518798848, + "allocated_bytes_peak_delta": 30864459264, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 83654339584, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 83654339584, + "device": 7, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 88803901440, + "reserved_bytes_peak": 88803901440, + "segment_count_current": 1548, + "segment_count_peak": 1548, + "timestamp_utc": "2026-08-26T19:12:49.424158Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116675051520, + "reserved_bytes_current_delta": 27871150080, + "reserved_bytes_peak": 116675051520, + "reserved_bytes_peak_delta": 27871150080, + "segment_count_current": 1852, + "segment_count_current_delta": 304, + "segment_count_peak": 1852, + "segment_count_peak_delta": 304, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T191250332363Z.pickle", + "timestamp_utc": "2026-08-26T19:12:50.368308Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191250332363Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191250332363Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a11770a176ab5666f981d10fd79b6e745d68bee --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191250332363Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c0ea2adfd73667c64932afaf4b0351267afcda9a42a5d95114c6c05e23151d1 +size 1826803 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191250686099Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191250686099Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a1aaca156ec05043fbcd225838640ce3a3c89917 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191250686099Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301043200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114526138880, + "active_bytes_peak_delta": 7340032, + "after": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114526138880, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114526138880, + "device": 7, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.685706Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301043200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114526138880, + "allocated_bytes_peak_delta": 7340032, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114518798848, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114518798848, + "device": 7, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.372206Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116675051520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116675051520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1852, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T191250686099Z.pickle", + "timestamp_utc": "2026-08-26T19:12:50.716045Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191250686099Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191250686099Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..148d8c2e5f4f7159f1cf31277cd7a72f71293d9d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191250686099Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f4a13f7a673a76d532ab59e6971a9bc056cfcaa38e2aba37e8515da538a322a +size 1815734 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191253369239Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191253369239Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8ccd7a34b12d19f1dd4e663779c12c76a5c46cc2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191253369239Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 1134592, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 7, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.368804Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 1134592, + "before": { + "active_bytes_current": 77301043200, + "active_bytes_peak": 114526138880, + "allocated_bytes_current": 77301043200, + "allocated_bytes_peak": 114526138880, + "device": 7, + "inactive_split_bytes_current": 1077915648, + "inactive_split_bytes_peak": 12728958464, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116675051520, + "reserved_bytes_peak": 116675051520, + "segment_count_current": 1852, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:50.719620Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 29093888, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 780140544, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 780140544, + "segment_count_current": 1763, + "segment_count_current_delta": -89, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T191253369239Z.pickle", + "timestamp_utc": "2026-08-26T19:12:53.407638Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191253369239Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191253369239Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..64a18c07b70d6dff449991fc42b3ba9fdabb832d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191253369239Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a45134c75e5dd57bcc88b4e594378c134dcf6d5f1ea8dffecd34269fcdd6056 +size 2123239 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191253712853Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191253712853Z.json new file mode 100644 index 0000000000000000000000000000000000000000..82dc508b639b4c6a42f8005b3c3623bc0e36ed92 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191253712853Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 7, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.712446Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 7, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.411789Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1763, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T191253712853Z.pickle", + "timestamp_utc": "2026-08-26T19:12:53.721987Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191253712853Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191253712853Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1dbd212b56c063b6749c565b76bb09a34e928415 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191253712853Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a36f56e16652bf750f0d7db78587e0e7053f32e170b19ea7e70a6b9bceaf9d35 +size 427803 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191254023828Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191254023828Z.json new file mode 100644 index 0000000000000000000000000000000000000000..079e93607bda7bc1cc17c88eec6c2eab608354fa --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191254023828Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318086656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114527273472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 7, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:54.023455Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318086656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114527273472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318086656, + "active_bytes_peak": 114527273472, + "allocated_bytes_current": 77318086656, + "allocated_bytes_peak": 114527273472, + "device": 7, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_peak": 12728958464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117455192064, + "reserved_bytes_peak": 117455192064, + "segment_count_current": 1763, + "segment_count_peak": 1852, + "timestamp_utc": "2026-08-26T19:12:53.723237Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1107009536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12728958464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117455192064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117455192064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1763, + "segment_count_current_delta": 0, + "segment_count_peak": 1852, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T191254023828Z.pickle", + "timestamp_utc": "2026-08-26T19:12:54.032832Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191254023828Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191254023828Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..abca3fe3e5d006251fd8657ae11dfe05068158d6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191254023828Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a87e4d4f9638217020bbb206dc5460ea38ccaea860b13b131022b571117ccf2f +size 427803 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/run.env b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..551c3cac71929b7408e30b8a933ef80fae2b8d07 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=QWEN3_235B +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-better-memory-dump-support +commit=c3a8a8a7 +num_layers=12 +hidden=4096 +ffn_hidden=12288 +moe_ffn_hidden=1536 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=18874368 +heads=64 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=12 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=65536 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.125 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=512 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/train.log b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..1c490a1f0beeba309299dba4928cdbab338c52df --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/train.log @@ -0,0 +1,1443 @@ +[19:11:54] WARN: ultraep-better-memory-dump-support lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +============================================================== + QWEN3_235B / ultraep (slots/rank=16 policy=matched) + E=128 K=8 H'=1536 m=16 + L=12 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.125 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ c3a8a8a7 +============================================================== +W0826 19:11:55.412000 630183 torch/distributed/run.py:766] +W0826 19:11:55.412000 630183 torch/distributed/run.py:766] ***************************************** +W0826 19:11:55.412000 630183 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:11:55.412000 630183 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 12 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 12288 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1536 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 16 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.125 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 12 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank 215 is connected to is connected to is connected to 777[Gloo] Rank peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : 3Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7[Gloo] Rank is connected to 77 + + +47 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7[Gloo] Rank Expected number of connected peer ranks is : +67 is connected to +7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank4]:[W826 19:12:13.742877470 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:12:13.742914331 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:12:13.751312340 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:12:13.751397339 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:12:13.751404928 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:12:13.751408685 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:12:13.751511148 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:12:13.899878714 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.769 seconds +time to initialize megatron (seconds): 6.869 +[after megatron is initialized] datetime: 2026-08-26 19:12:19 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 8521882624 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 14 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.final_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight +Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight +Params for bucket 4 (71835904 elements, 71835904 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight +Params for bucket 5 (71835904 elements, 71835904 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.q_layernorm.weight +Params for bucket 6 (71835904 elements, 71835904 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.q_layernorm.weight +Params for bucket 7 (71835904 elements, 71835904 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.pre_mlp_layernorm.weight +Params for bucket 8 (71835904 elements, 71835904 padded size): + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight +Params for bucket 9 (71835904 elements, 71835904 padded size): + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 10 (71835904 elements, 71835904 padded size): + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 11 (71835904 elements, 71835904 padded size): + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight +Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight +Params for bucket 13 (71835904 elements, 71835904 padded size): + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 14 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 +Params for bucket 4 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 +Params for bucket 5 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 +Params for bucket 6 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 +Params for bucket 9 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 +Params for bucket 10 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 +Params for bucket 11 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 +Params for bucket 12 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 +Params for bucket 15 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 +Params for bucket 16 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 +Params for bucket 17 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 +Params for bucket 18 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 +Params for bucket 23 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 +Params for bucket 24 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 +Params for bucket 25 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 +Params for bucket 28 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 +Params for bucket 29 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 +Params for bucket 30 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 +Params for bucket 31 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 +Params for bucket 34 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 +Params for bucket 35 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 +Params for bucket 36 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 +Params for bucket 37 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 +Params for bucket 42 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 +Params for bucket 43 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 44 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 +Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 +Params for bucket 48 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 +Params for bucket 49 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 +Params for bucket 50 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 +Params for bucket 54 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 +Params for bucket 55 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 56 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 +Params for bucket 61 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 +Params for bucket 62 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 +Params for bucket 63 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 66 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 +Params for bucket 67 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 +Params for bucket 68 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 +Params for bucket 69 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 70 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 +Params for bucket 71 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 +Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 +Params for bucket 73 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 +Params for bucket 74 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 +Params for bucket 75 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 +Params for bucket 76 (37748736 elements, 37748736 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:12:23 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:12:23 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3730.92, 3740.09) + train/valid/test-data-iterators-setup ..........: (68.70, 90.68) +[before the start of training step] datetime: 2026-08-26 19:12:23 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:12:49] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 25882.8 | throughput per GPU (TFLOP/s/GPU): 7.0 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.116001E+01 | loss scale: 1.0 | grad norm: 44.807 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67456 | min safe capacity: 1.0293 | +Number of parameters in transformer block in billions: 29.85 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 30.26 +Number of parameters in most loaded shard in billions: 30.2590 +Theoretical memory footprints: weight and optimizer=216428.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 73717.49853515625 | max allocated: 79776.84912109375 | reserved: 84758.0 | max reserved: 84758.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 19:12:50] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 951.8 | throughput per GPU (TFLOP/s/GPU): 190.0 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117607E+01 | loss scale: 1.0 | grad norm: 44.339 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67712 | min safe capacity: 1.0332 | + [2026-08-26 19:12:50] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 353.8 | throughput per GPU (TFLOP/s/GPU): 511.1 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.685659E+00 | loss scale: 1.0 | grad norm: 62.790 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 19:12:53] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 2682.9 | throughput per GPU (TFLOP/s/GPU): 67.4 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338081E+01 | loss scale: 1.0 | grad norm: 20.306 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | + [2026-08-26 19:12:53] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 343.7 | throughput per GPU (TFLOP/s/GPU): 526.1 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.050303E+01 | loss scale: 1.0 | grad norm: 95.235 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | + [2026-08-26 19:12:54] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 310.9 | throughput per GPU (TFLOP/s/GPU): 581.5 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.040408E+01 | loss scale: 1.0 | grad norm: 49.659 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +[after training is done] datetime: 2026-08-26 19:12:54 +W0826 19:12:54.087000 630183 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:12:54.087000 630183 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 630274 closing signal SIGTERM +W0826 19:12:54.098000 630183 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 630275 closing signal SIGTERM +W0826 19:12:54.102000 630183 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 630276 closing signal SIGTERM +W0826 19:12:54.110000 630183 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 630277 closing signal SIGTERM +W0826 19:12:54.120000 630183 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 630278 closing signal SIGTERM +W0826 19:12:54.128000 630183 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 630279 closing signal SIGTERM +W0826 19:12:54.142000 630183 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 630280 closing signal SIGTERM +W0826 19:12:54.149000 630183 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 630281 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 630183 got signal: 15 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..e8d08f0af583f0b0a609398eddba10c850c4caa1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 2 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2551 +1, 2551 +2, 2551 +3, 2551 +4, 2551 +5, 2551 +6, 2551 +7, 2551 +0, 2959 +1, 2959 +2, 2959 +3, 2959 +4, 2959 +5, 2959 +6, 2959 +7, 2959 +0, 51539 +1, 51539 +2, 51539 +3, 51539 +4, 51251 +5, 51603 +6, 51539 +7, 51539 +0, 53017 +1, 53017 +2, 53017 +3, 53017 +4, 52729 +5, 53017 +6, 53017 +7, 53017 +0, 54651 +1, 54651 +2, 54651 +3, 54651 +4, 54363 +5, 54651 +6, 54651 +7, 54651 +0, 54651 +1, 54651 +2, 54651 +3, 54651 +4, 54363 +5, 54651 +6, 54651 +7, 54651 +0, 55291 +1, 55291 +2, 55291 +3, 55291 +4, 55003 +5, 55291 +6, 55291 +7, 55291 +0, 57203 +1, 57203 +2, 57203 +3, 57203 +4, 56915 +5, 57203 +6, 57203 +7, 57203 +0, 84021 +1, 84163 +2, 84017 +3, 84203 +4, 84123 +5, 84121 +6, 84245 +7, 84183 +0, 91083 +1, 91131 +2, 91131 +3, 91131 +4, 91123 +5, 91131 +6, 91131 +7, 91131 +0, 91083 +1, 91131 +2, 91131 +3, 91131 +4, 91123 +5, 91131 +6, 91131 +7, 91131 +0, 91083 +1, 91131 +2, 91131 +3, 91131 +4, 91123 +5, 91131 +6, 91131 +7, 91131 +0, 91447 +1, 91535 +2, 91467 +3, 91487 +4, 91433 +5, 91465 +6, 91517 +7, 91511 +0, 95929 +1, 96055 +2, 96043 +3, 95987 +4, 96099 +5, 95995 +6, 96037 +7, 95973 +0, 105409 +1, 108041 +2, 106921 +3, 106921 +4, 106985 +5, 106921 +6, 106921 +7, 106921 +0, 90555 +1, 87383 +2, 87245 +3, 88799 +4, 87725 +5, 87533 +6, 87879 +7, 87409 +0, 123481 +1, 123693 +2, 123693 +3, 123693 +4, 123775 +5, 123693 +6, 123693 +7, 123693 +0, 119151 +1, 119363 +2, 119363 +3, 119363 +4, 119445 +5, 119363 +6, 119363 +7, 119363 +0, 117083 +1, 117683 +2, 117683 +3, 117295 +4, 117765 +5, 117683 +6, 117295 +7, 117683 +0, 4 +1, 115649 +2, 115797 +3, 4 +4, 115751 +5, 4 +6, 4 +7, 115645 +0, 1 +1, 1 +2, 1 +3, 1 +4, 115692 +5, 1 +6, 3 +7, 4 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191355494533Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191355494533Z.json new file mode 100644 index 0000000000000000000000000000000000000000..62d914673de2af71cf1b174945a0576812cc8905 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191355494533Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77298399744, + "active_bytes_current_delta": 32777599488, + "active_bytes_peak": 84819154432, + "active_bytes_peak_delta": 40273188352, + "after": { + "active_bytes_current": 77298399744, + "active_bytes_peak": 84819154432, + "allocated_bytes_current": 77298399744, + "allocated_bytes_peak": 84819154432, + "device": 0, + "inactive_split_bytes_current": 1049101824, + "inactive_split_bytes_peak": 11031815680, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 90475331584, + "reserved_bytes_peak": 90475331584, + "segment_count_current": 1687, + "segment_count_peak": 1687, + "timestamp_utc": "2026-08-26T19:13:55.494032Z", + "world_size": 8 + }, + "allocated_bytes_current": 77298399744, + "allocated_bytes_current_delta": 32777599488, + "allocated_bytes_peak": 84819154432, + "allocated_bytes_peak_delta": 40273188352, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 0, + "inactive_split_bytes_current": 446332928, + "inactive_split_bytes_peak": 1050312704, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:13:28.769856Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1049101824, + "inactive_split_bytes_current_delta": 602768896, + "inactive_split_bytes_peak": 11031815680, + "inactive_split_bytes_peak_delta": 9981502976, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 90475331584, + "reserved_bytes_current_delta": 42224058368, + "reserved_bytes_peak": 90475331584, + "reserved_bytes_peak_delta": 42224058368, + "segment_count_current": 1687, + "segment_count_current_delta": 1066, + "segment_count_peak": 1687, + "segment_count_peak_delta": 1066, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T191355494533Z.pickle", + "timestamp_utc": "2026-08-26T19:13:55.534187Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191355494533Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191355494533Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf2fb9795e22fc89eac032cd7288a9ae38c9776e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191355494533Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b59b043502d0cf68f7bca0b29081af4401aa3afd9f7f7010254adbbbb1fcb81c +size 2313666 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191356330627Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191356330627Z.json new file mode 100644 index 0000000000000000000000000000000000000000..afe15bdf6c7b44eed127b16a01b8fbcc9de4ef62 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191356330627Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77298399744, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115685710848, + "active_bytes_peak_delta": 30866556416, + "after": { + "active_bytes_current": 77298399744, + "active_bytes_peak": 115685710848, + "allocated_bytes_current": 77298399744, + "allocated_bytes_peak": 115685710848, + "device": 0, + "inactive_split_bytes_current": 1051198976, + "inactive_split_bytes_peak": 11031815680, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117918662656, + "reserved_bytes_peak": 117918662656, + "segment_count_current": 1861, + "segment_count_peak": 1861, + "timestamp_utc": "2026-08-26T19:13:56.330208Z", + "world_size": 8 + }, + "allocated_bytes_current": 77298399744, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115685710848, + "allocated_bytes_peak_delta": 30866556416, + "before": { + "active_bytes_current": 77298399744, + "active_bytes_peak": 84819154432, + "allocated_bytes_current": 77298399744, + "allocated_bytes_peak": 84819154432, + "device": 0, + "inactive_split_bytes_current": 1049101824, + "inactive_split_bytes_peak": 11031815680, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 90475331584, + "reserved_bytes_peak": 90475331584, + "segment_count_current": 1687, + "segment_count_peak": 1687, + "timestamp_utc": "2026-08-26T19:13:55.538947Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1051198976, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11031815680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117918662656, + "reserved_bytes_current_delta": 27443331072, + "reserved_bytes_peak": 117918662656, + "reserved_bytes_peak_delta": 27443331072, + "segment_count_current": 1861, + "segment_count_current_delta": 174, + "segment_count_peak": 1861, + "segment_count_peak_delta": 174, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T191356330627Z.pickle", + "timestamp_utc": "2026-08-26T19:13:56.366937Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191356330627Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191356330627Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3438f12ff46b228b890b4b3d54947e3f84b3ac69 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191356330627Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1f749649f85ba1d7c0284bb40a4034ece190ed4234aa60a98f189f226cc9193 +size 1837837 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191356688278Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191356688278Z.json new file mode 100644 index 0000000000000000000000000000000000000000..136292827eb16b421862b8569f8aca84dff7d61d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191356688278Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77298399744, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115685989376, + "active_bytes_peak_delta": 278528, + "after": { + "active_bytes_current": 77298399744, + "active_bytes_peak": 115685989376, + "allocated_bytes_current": 77298399744, + "allocated_bytes_peak": 115685989376, + "device": 0, + "inactive_split_bytes_current": 1051198976, + "inactive_split_bytes_peak": 11031815680, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117918662656, + "reserved_bytes_peak": 117918662656, + "segment_count_current": 1861, + "segment_count_peak": 1861, + "timestamp_utc": "2026-08-26T19:13:56.687856Z", + "world_size": 8 + }, + "allocated_bytes_current": 77298399744, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115685989376, + "allocated_bytes_peak_delta": 278528, + "before": { + "active_bytes_current": 77298399744, + "active_bytes_peak": 115685710848, + "allocated_bytes_current": 77298399744, + "allocated_bytes_peak": 115685710848, + "device": 0, + "inactive_split_bytes_current": 1051198976, + "inactive_split_bytes_peak": 11031815680, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117918662656, + "reserved_bytes_peak": 117918662656, + "segment_count_current": 1861, + "segment_count_peak": 1861, + "timestamp_utc": "2026-08-26T19:13:56.371067Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1051198976, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11031815680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117918662656, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117918662656, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1861, + "segment_count_current_delta": 0, + "segment_count_peak": 1861, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T191356688278Z.pickle", + "timestamp_utc": "2026-08-26T19:13:56.719591Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191356688278Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191356688278Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..328abb54dcb9501fbaac170d1668eff11625e1a1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191356688278Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cde166ee394e00d85af5b3b3b15069d674c6e398dae02124d65062fdc7c9a19d +size 1827594 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191359327377Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191359327377Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d2887af2148f111b94bcc6884fb2679d085632ec --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191359327377Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77315443200, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 115693415424, + "active_bytes_peak_delta": 7426048, + "after": { + "active_bytes_current": 77315443200, + "active_bytes_peak": 115693415424, + "allocated_bytes_current": 77315443200, + "allocated_bytes_peak": 115693415424, + "device": 0, + "inactive_split_bytes_current": 1057224192, + "inactive_split_bytes_peak": 11031815680, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118805757952, + "reserved_bytes_peak": 118805757952, + "segment_count_current": 1683, + "segment_count_peak": 1861, + "timestamp_utc": "2026-08-26T19:13:59.326914Z", + "world_size": 8 + }, + "allocated_bytes_current": 77315443200, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115693415424, + "allocated_bytes_peak_delta": 7426048, + "before": { + "active_bytes_current": 77298399744, + "active_bytes_peak": 115685989376, + "allocated_bytes_current": 77298399744, + "allocated_bytes_peak": 115685989376, + "device": 0, + "inactive_split_bytes_current": 1051198976, + "inactive_split_bytes_peak": 11031815680, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117918662656, + "reserved_bytes_peak": 117918662656, + "segment_count_current": 1861, + "segment_count_peak": 1861, + "timestamp_utc": "2026-08-26T19:13:56.723379Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1057224192, + "inactive_split_bytes_current_delta": 6025216, + "inactive_split_bytes_peak": 11031815680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118805757952, + "reserved_bytes_current_delta": 887095296, + "reserved_bytes_peak": 118805757952, + "reserved_bytes_peak_delta": 887095296, + "segment_count_current": 1683, + "segment_count_current_delta": -178, + "segment_count_peak": 1861, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T191359327377Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.365375Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191359327377Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191359327377Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e3fb159adbf3cc13cb94177c1f25b4f1c020134b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191359327377Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26c43c8f8ef5df5d6e2c5978ae38eabe6316cceb272171d3478b1f1adb5bf991 +size 2185511 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191359673283Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191359673283Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4a95b6d4a4e20bd7d56b67dd77e62614bb06b4e2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191359673283Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77315443200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115693415424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77315443200, + "active_bytes_peak": 115693415424, + "allocated_bytes_current": 77315443200, + "allocated_bytes_peak": 115693415424, + "device": 0, + "inactive_split_bytes_current": 1057224192, + "inactive_split_bytes_peak": 11031815680, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118805757952, + "reserved_bytes_peak": 118805757952, + "segment_count_current": 1683, + "segment_count_peak": 1861, + "timestamp_utc": "2026-08-26T19:13:59.672899Z", + "world_size": 8 + }, + "allocated_bytes_current": 77315443200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115693415424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77315443200, + "active_bytes_peak": 115693415424, + "allocated_bytes_current": 77315443200, + "allocated_bytes_peak": 115693415424, + "device": 0, + "inactive_split_bytes_current": 1057224192, + "inactive_split_bytes_peak": 11031815680, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118805757952, + "reserved_bytes_peak": 118805757952, + "segment_count_current": 1683, + "segment_count_peak": 1861, + "timestamp_utc": "2026-08-26T19:13:59.369504Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1057224192, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11031815680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118805757952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118805757952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1683, + "segment_count_current_delta": 0, + "segment_count_peak": 1861, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T191359673283Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.681787Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191359673283Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191359673283Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..31dbe0a1e716a277b3009328d60ab5d4ee23c686 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191359673283Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ecad432860da0cb9629495e9f6841fc86b044550305775ddb2225213a63e1ba +size 430319 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191359985715Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191359985715Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8a2f4c7efd9a4c9bc9d8fe65e03f9ee2f360a760 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191359985715Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77315443200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115693415424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77315443200, + "active_bytes_peak": 115693415424, + "allocated_bytes_current": 77315443200, + "allocated_bytes_peak": 115693415424, + "device": 0, + "inactive_split_bytes_current": 1057224192, + "inactive_split_bytes_peak": 11031815680, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118805757952, + "reserved_bytes_peak": 118805757952, + "segment_count_current": 1683, + "segment_count_peak": 1861, + "timestamp_utc": "2026-08-26T19:13:59.985343Z", + "world_size": 8 + }, + "allocated_bytes_current": 77315443200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115693415424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77315443200, + "active_bytes_peak": 115693415424, + "allocated_bytes_current": 77315443200, + "allocated_bytes_peak": 115693415424, + "device": 0, + "inactive_split_bytes_current": 1057224192, + "inactive_split_bytes_peak": 11031815680, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118805757952, + "reserved_bytes_peak": 118805757952, + "segment_count_current": 1683, + "segment_count_peak": 1861, + "timestamp_utc": "2026-08-26T19:13:59.682972Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1057224192, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11031815680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118805757952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118805757952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1683, + "segment_count_current_delta": 0, + "segment_count_peak": 1861, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T191359985715Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.994153Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191359985715Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191359985715Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..85fc72155a1a44271c36b27dcde6c556eeb60c73 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191359985715Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0368fcbc5889cc61aa8003d5c76f5d0338365cdd9903ae673aa830bdcae778f6 +size 430288 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191355494311Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191355494311Z.json new file mode 100644 index 0000000000000000000000000000000000000000..025dbe3b5ac90ce32e157b23f6e42559e009a57c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191355494311Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 32780593664, + "active_bytes_peak": 84822148608, + "active_bytes_peak_delta": 40276182528, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 84822148608, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 84822148608, + "device": 1, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 90550829056, + "reserved_bytes_peak": 90550829056, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-26T19:13:55.493801Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 32780593664, + "allocated_bytes_peak": 84822148608, + "allocated_bytes_peak_delta": 40276182528, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 1, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:13:28.769747Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 849335808, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 10484395520, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 90550829056, + "reserved_bytes_current_delta": 42299555840, + "reserved_bytes_peak": 90550829056, + "reserved_bytes_peak_delta": 42299555840, + "segment_count_current": 1692, + "segment_count_current_delta": 1071, + "segment_count_peak": 1692, + "segment_count_peak_delta": 1071, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T191355494311Z.pickle", + "timestamp_utc": "2026-08-26T19:13:55.534357Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191355494311Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191355494311Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44909bf84d5160e2e47e14ca6f9d3098839e8782 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191355494311Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e1c57466a18dae92a857cbdb649e8da129915d42e874effdd418d08637af95b +size 2301742 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191356330470Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191356330470Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2e68f4c2fdb8cfaeb75e3edea039838239b1ad8b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191356330470Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115689589760, + "active_bytes_peak_delta": 30867441152, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689589760, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689589760, + "device": 1, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.330028Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115689589760, + "allocated_bytes_peak_delta": 30867441152, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 84822148608, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 84822148608, + "device": 1, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 90550829056, + "reserved_bytes_peak": 90550829056, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-26T19:13:55.538477Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118057074688, + "reserved_bytes_current_delta": 27506245632, + "reserved_bytes_peak": 118057074688, + "reserved_bytes_peak_delta": 27506245632, + "segment_count_current": 1871, + "segment_count_current_delta": 179, + "segment_count_peak": 1871, + "segment_count_peak_delta": 179, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T191356330470Z.pickle", + "timestamp_utc": "2026-08-26T19:13:56.367196Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191356330470Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191356330470Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1fc41f075f77cb3b6fe79a87220746764b0be824 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191356330470Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db4269081572b4427c4ce644164ff9cd4157b685e8ef6c2ca7b78b298a80b79b +size 1827611 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191356688096Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191356688096Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e93515802a2fc68adccb44d614b204efae11642f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191356688096Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115689851904, + "active_bytes_peak_delta": 262144, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689851904, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689851904, + "device": 1, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.687694Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115689851904, + "allocated_bytes_peak_delta": 262144, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689589760, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689589760, + "device": 1, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.371119Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118057074688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118057074688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1871, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T191356688096Z.pickle", + "timestamp_utc": "2026-08-26T19:13:56.719221Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191356688096Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191356688096Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3f2d4451877242ae67abac2bf16122f139e0d493 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191356688096Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca9e2b8fda9663be171df869b15947c2bca36701e275435a6d9e73671d7b37ab +size 1818569 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191359327197Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191359327197Z.json new file mode 100644 index 0000000000000000000000000000000000000000..afaf35fdb4d883c2c970a483da74af91e5f00ef4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191359327197Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 6557696, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 1, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.326762Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 6557696, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689851904, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689851904, + "device": 1, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.722733Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 8122368, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 970981376, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 970981376, + "segment_count_current": 1688, + "segment_count_current_delta": -183, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T191359327197Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.364810Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191359327197Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191359327197Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f99f419c12ae0c8cd313eabda7e7cbc36c927f8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191359327197Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be770f9d8e969bc10f16ba70d88c9bdabadf2b5b6a36fcfa7869749100f5eda1 +size 2188969 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191359673133Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191359673133Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dfe3b7c24f88ef0c80647d6087f2712ba700371d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191359673133Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 1, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.672753Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 1, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.368825Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1688, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T191359673133Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.681533Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191359673133Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191359673133Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8a476c9c6b32d466605d5bc6c6c633959117974c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191359673133Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fec41b8ee2a4fa7dcd45cc2fac16080185f42b48a1a4a24cb96e82f10e47044f +size 421721 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191359985607Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191359985607Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c294d79425236fdc82f4532462ac9d96c18600f9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191359985607Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 1, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.985241Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 1, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.682618Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1688, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T191359985607Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.994061Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191359985607Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191359985607Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..21ba85e62655a22a40b8edae2ff1def6a92f7252 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191359985607Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beaabe5219e267ed5c5b18bab2891810749b46e8e24769f1cc512bc5d7e300e2 +size 421721 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191355494413Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191355494413Z.json new file mode 100644 index 0000000000000000000000000000000000000000..93692dda1850033c1fdec20224f223a65b40d831 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191355494413Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 32780593664, + "active_bytes_peak": 84822148608, + "active_bytes_peak_delta": 40276182528, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 84822148608, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 84822148608, + "device": 2, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 90550829056, + "reserved_bytes_peak": 90550829056, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-26T19:13:55.493933Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 32780593664, + "allocated_bytes_peak": 84822148608, + "allocated_bytes_peak_delta": 40276182528, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 2, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:13:28.769752Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 849335808, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 10484395520, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 90550829056, + "reserved_bytes_current_delta": 42299555840, + "reserved_bytes_peak": 90550829056, + "reserved_bytes_peak_delta": 42299555840, + "segment_count_current": 1692, + "segment_count_current_delta": 1071, + "segment_count_peak": 1692, + "segment_count_peak_delta": 1071, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T191355494413Z.pickle", + "timestamp_utc": "2026-08-26T19:13:55.535059Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191355494413Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191355494413Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ec54e6941997191f48537cb36d06c73579140fc4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191355494413Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5b7f256af21d8d67af63c829bb805e82b399de701fb8f7fe0e98de904024382 +size 2301744 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191356330610Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191356330610Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4fc3ff4f0a71ec95e28420872225396f25daf561 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191356330610Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115689589760, + "active_bytes_peak_delta": 30867441152, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689589760, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689589760, + "device": 2, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.330167Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115689589760, + "allocated_bytes_peak_delta": 30867441152, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 84822148608, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 84822148608, + "device": 2, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 90550829056, + "reserved_bytes_peak": 90550829056, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-26T19:13:55.539285Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118057074688, + "reserved_bytes_current_delta": 27506245632, + "reserved_bytes_peak": 118057074688, + "reserved_bytes_peak_delta": 27506245632, + "segment_count_current": 1871, + "segment_count_current_delta": 179, + "segment_count_peak": 1871, + "segment_count_peak_delta": 179, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T191356330610Z.pickle", + "timestamp_utc": "2026-08-26T19:13:56.368480Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191356330610Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191356330610Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2bbe605f1f39574aaa547e1b1dccafaebf2d6b64 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191356330610Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62572f76292794b638b4b09609c002894f5430485ccd7a979fb7bdda0814ce7c +size 1827611 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191356688291Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191356688291Z.json new file mode 100644 index 0000000000000000000000000000000000000000..742bed6f7c18f44d979e69ed329ba02a47636a2d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191356688291Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115689851904, + "active_bytes_peak_delta": 262144, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689851904, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689851904, + "device": 2, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.687869Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115689851904, + "allocated_bytes_peak_delta": 262144, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689589760, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689589760, + "device": 2, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.372649Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118057074688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118057074688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1871, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T191356688291Z.pickle", + "timestamp_utc": "2026-08-26T19:13:56.719541Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191356688291Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191356688291Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..191921807cb2d39aaf045c3709a01f2dd7576260 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191356688291Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08973356f0f2a655993c552035a3d66b8acea653f26f13f81d08459ec864524a +size 1818571 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191359327367Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191359327367Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7064808e661c97f4eebf505a333ff72761727484 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191359327367Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 6557696, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 2, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.326885Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 6557696, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689851904, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689851904, + "device": 2, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.723338Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 8122368, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 970981376, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 970981376, + "segment_count_current": 1688, + "segment_count_current_delta": -183, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T191359327367Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.365482Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191359327367Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191359327367Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5dd32afe0064f67abc3dc6d01fae6764d029a6ac --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191359327367Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f63398ce26be494c9e46c683fe19b4a43d9c9791a4760e80213b139916823eb6 +size 2188971 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191359673250Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191359673250Z.json new file mode 100644 index 0000000000000000000000000000000000000000..34cf9f846a130cba0d163fa5bf71f1fb4570fc8d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191359673250Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 2, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.672859Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 2, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.369621Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1688, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T191359673250Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.681648Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191359673250Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191359673250Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6f4785e568d54519cb58012478ec25e77f4e8bc1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191359673250Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a8372dc5f55e483b106e70ad172e2968fdfdb5f286e06b39ba27f05859307bd +size 421723 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191359985735Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191359985735Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6ef51626b17dbe4a83065e6bec393aa740ecab84 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191359985735Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 2, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.985352Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 2, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.682805Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1688, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T191359985735Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.994208Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191359985735Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191359985735Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ab002761e4490d83cd350ac81c44901e0a71e971 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191359985735Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f983b5a6f418b72938f616873ad1204fc305c3bb1f61bcaf015b5f0e952882b +size 421723 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191355494417Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191355494417Z.json new file mode 100644 index 0000000000000000000000000000000000000000..10d1f5581bba9fa7e31d7f6575c115d3adedda92 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191355494417Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 32780593664, + "active_bytes_peak": 84822148608, + "active_bytes_peak_delta": 40276182528, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 84822148608, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 84822148608, + "device": 3, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 90550829056, + "reserved_bytes_peak": 90550829056, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-26T19:13:55.493933Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 32780593664, + "allocated_bytes_peak": 84822148608, + "allocated_bytes_peak_delta": 40276182528, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 3, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:13:28.769773Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 849335808, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 10484395520, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 90550829056, + "reserved_bytes_current_delta": 42299555840, + "reserved_bytes_peak": 90550829056, + "reserved_bytes_peak_delta": 42299555840, + "segment_count_current": 1692, + "segment_count_current_delta": 1071, + "segment_count_peak": 1692, + "segment_count_peak_delta": 1071, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T191355494417Z.pickle", + "timestamp_utc": "2026-08-26T19:13:55.533706Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191355494417Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191355494417Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..80d21a004af251c7b6fa46d9e7a80f6eb819313e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191355494417Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb34d5cec1e76189269b06e050f408a688c5610bbcf5c178ee1c44946996e3c5 +size 2301744 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191356330597Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191356330597Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5e7ee2a5462a24d7c37f5789ae5bdec90c14f615 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191356330597Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115689589760, + "active_bytes_peak_delta": 30867441152, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689589760, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689589760, + "device": 3, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.330154Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115689589760, + "allocated_bytes_peak_delta": 30867441152, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 84822148608, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 84822148608, + "device": 3, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 90550829056, + "reserved_bytes_peak": 90550829056, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-26T19:13:55.537872Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118057074688, + "reserved_bytes_current_delta": 27506245632, + "reserved_bytes_peak": 118057074688, + "reserved_bytes_peak_delta": 27506245632, + "segment_count_current": 1871, + "segment_count_current_delta": 179, + "segment_count_peak": 1871, + "segment_count_peak_delta": 179, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T191356330597Z.pickle", + "timestamp_utc": "2026-08-26T19:13:56.367470Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191356330597Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191356330597Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..40c85d70e020ab282710d94339e9bb7cb96fe7af --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191356330597Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd5a9abb3a0a4eae239bd846c170d94795adf34646efac9219d15e8739c58ba8 +size 1827609 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191356688257Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191356688257Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1b40cfd9891f63e4b47b61815312409e74bdc905 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191356688257Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115689851904, + "active_bytes_peak_delta": 262144, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689851904, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689851904, + "device": 3, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.687837Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115689851904, + "allocated_bytes_peak_delta": 262144, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689589760, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689589760, + "device": 3, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.371567Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118057074688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118057074688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1871, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T191356688257Z.pickle", + "timestamp_utc": "2026-08-26T19:13:56.719426Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191356688257Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191356688257Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b53ae4f354c1b662ef040c9361056b0ac015bcf0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191356688257Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e0c40407565eb80a4cc661a34ddf5aa79f56a49275c1ac096aedf8cc36de467 +size 1818571 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191359327379Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191359327379Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d24a72845170a120cf1f2f888bb5c327d09f6f74 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191359327379Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 6557696, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 3, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.326898Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 6557696, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689851904, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689851904, + "device": 3, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.723133Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 8122368, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 970981376, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 970981376, + "segment_count_current": 1688, + "segment_count_current_delta": -183, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T191359327379Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.365647Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191359327379Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191359327379Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cfe6c51edd07f58beec4f0fb2dc7265d1366b538 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191359327379Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ca9d5ea43e22acfe905c57d49b5093e01c1d61701d4c2d190c774ef202a9a88 +size 2188971 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191359673365Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191359673365Z.json new file mode 100644 index 0000000000000000000000000000000000000000..21003d348a98e4c3bd577a0a122029eb6a32a077 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191359673365Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 3, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.672872Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 3, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.369921Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1688, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T191359673365Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.681987Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file