diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000-merged/preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7df5b45379044a680218cb2b7ed4d01a6633f537 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000-merged/preprocessor_config.json @@ -0,0 +1,30 @@ +{ + "dither": 0.0, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_pixels": 12845056, + "merge_size": 2, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "return_attention_mask": true, + "sampling_rate": 16000, + "temporal_patch_size": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000-merged/special_tokens_map.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000-merged/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..fca9909b535ff91b68474e5ee7258e1574e60d1e --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000-merged/special_tokens_map.json @@ -0,0 +1,44 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "image_token": "<|image_pad|>", + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000-merged/tokenizer_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1491f11cc7fc66a763bcda6f4f6b8c3d026845 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000-merged/tokenizer_config.json @@ -0,0 +1,317 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151669": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151670": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151675": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "image_token": "<|image_pad|>", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3OmniMoeProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000-merged/video_preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000-merged/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6e0fb77bf83924807f06b6f314210795f76c2e42 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000-merged/video_preprocessor_config.json @@ -0,0 +1,54 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "dither": 0.0, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": false, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "fps": null, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "max_pixels": 12845056, + "merge_size": 2, + "min_frames": 4, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "num_frames": null, + "pad_size": null, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_attention_mask": true, + "return_metadata": false, + "sampling_rate": 16000, + "size": { + "longest_edge": 12845056, + "shortest_edge": 3136 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen2VLVideoProcessor" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000/iter_0015000/.metadata b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000/iter_0015000/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..4f67db7894ca9a8f9bfefa4c131029b498c8306f Binary files /dev/null and b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000/iter_0015000/.metadata differ diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000/iter_0015000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000/iter_0015000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a41c3e7d25150d3305e868a8250a1244533a0d4 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000/iter_0015000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bff5b41f35f78fd6c92a3431c47e4dbbb901ab919f9c474ace015f952fe1c20 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000/iter_0015000/metadata.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000/iter_0015000/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000/iter_0015000/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..f0cc45cd73868e30a4b551c408cba9bcba24dfe0 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-15000/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +15000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/added_tokens.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..c962f0ea74c2dc3172f28f532b632288bf7d56c3 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/added_tokens.json @@ -0,0 +1,35 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "": 151671, + "": 151672, + "": 151674, + "": 151673, + "<|audio_end|>": 151670, + "<|audio_pad|>": 151675, + "<|audio_start|>": 151669, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/chat_template.jinja b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1efcce1402ae5d1d61d4a4cf56accea5921251cf --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/chat_template.jinja @@ -0,0 +1,122 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- "<|vision_start|><|image_pad|><|vision_end|>" }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- "<|audio_start|><|audio_pad|><|audio_end|>" }} + {%- elif content.type == 'video' or 'video' in content %} + {{- "<|vision_start|><|video_pad|><|vision_end|>" }} + {%- elif content.type == 'text' %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} + {{- '\n\n' }} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|image_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- '<|im_start|>system\n' +"<|audio_start|><|audio_pad|><|audio_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'video' or 'video' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|video_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'text' %} + {{- '<|im_start|>system\n' +content.text+ '<|im_end|>\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = namespace(text="") %} + {%- for mcontent in message.content %} + {%- if mcontent.type == 'image' or 'image' in mcontent or 'image_url' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|image_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'audio' or 'audio' in mcontent or 'audio_url' in mcontent %} + {%- set content.text = content.text~"<|audio_start|><|audio_pad|><|audio_end|>" %} + {%- elif mcontent.type == 'video' or 'video' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|video_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'text' %} + {%- set content.text = content.text~mcontent.text %} + {%- endif %} + {%- endfor %} + {%- set content = content.text %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = "" %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip("\n") + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %}{{- '\n' }}{%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}{{- '<|im_start|>user' }}{%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}{{- '<|im_end|>\n' }}{%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %}{{- '\n\n\n\n' }}{%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6dce3d213c49a2ea04a5bc9ba76b53e440cab76c --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/config.json @@ -0,0 +1,552 @@ +{ + "architectures": [ + "Qwen3OmniMoeForConditionalGeneration" + ], + "assistant_token_id": 77091, + "code2wav_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "codebook_dim": 512, + "codebook_size": 2048, + "decoder_dim": 1536, + "hidden_act": "silu", + "hidden_size": 1024, + "intermediate_size": 3072, + "layer_scale_initial_scale": 0.01, + "max_position_embeddings": 8000, + "model_type": "", + "num_attention_heads": 16, + "num_hidden_layers": 8, + "num_key_value_heads": 16, + "num_quantizers": 16, + "num_semantic_quantizers": 1, + "rms_norm_eps": 1e-05, + "rope_theta": 10000, + "semantic_codebook_size": 4096, + "sliding_window": 72, + "upsample_rates": [ + 8, + 5, + 4, + 3 + ], + "upsampling_ratios": [ + 2, + 2 + ], + "vector_quantization_hidden_dimension": 512 + }, + "dtype": "bfloat16", + "enable_audio_output": true, + "hidden_size": 1024, + "im_end_token_id": 151645, + "im_start_token_id": 151644, + "keys_to_ignore_at_inference": [ + "past_key_values" + ], + "model_type": "qwen3_omni_moe", + "system_token_id": 8948, + "talker_config": { + "accept_hidden_layer": 24, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "code_predictor_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": null, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 3072, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 32768, + "max_window_layers": 28, + "min_length": 0, + "model_type": "qwen3_omni_moe_talker_code_predictor", + "no_repeat_ngram_size": 0, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_code_groups": 16, + "num_hidden_layers": 5, + "num_key_value_heads": 8, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sep_token_id": null, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 2048 + }, + "codec_bos_id": 2149, + "codec_eos_token_id": 2150, + "codec_nothink_id": 2155, + "codec_pad_id": 2148, + "codec_think_bos_id": 2156, + "codec_think_eos_id": 2157, + "image_token_id": 151655, + "model_type": "", + "num_code_groups": 16, + "output_router_logits": false, + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "spatial_merge_size": 2, + "speaker_id": { + "aiden": 2303, + "chelsie": 2301, + "ethan": 2302 + }, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 2048, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_talker_text", + "moe_intermediate_size": 384, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 6, + "num_hidden_layers": 20, + "num_key_value_heads": 2, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 768, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 3072 + }, + "thinker_hidden_size": 2048, + "video_token_id": 151656, + "vision_start_token_id": 151652 + }, + "thinker_config": { + "audio_config": { + "_name_or_path": "", + "activation_dropout": 0, + "activation_function": "gelu", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "conv_chunksize": 500, + "cross_attention_hidden_size": null, + "d_model": 1280, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "downsample_hidden_size": 480, + "dropout": 0, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layers": 32, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_source_positions": 1500, + "min_length": 0, + "model_type": "qwen3_omni_moe_audio_encoder", + "n_window": 50, + "n_window_infer": 800, + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 32, + "num_mel_bins": 128, + "num_return_sequences": 1, + "output_attentions": false, + "output_dim": 2048, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "scale_embedding": false, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "dtype": "bfloat16", + "image_token_id": 151655, + "initializer_range": 0.02, + "model_type": "qwen3_omni_moe_thinker", + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0.0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 768, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_text", + "moe_intermediate_size": 768, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 32, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 48, + "num_key_value_heads": 4, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 0, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_qk_norm": true, + "use_sliding_window": false, + "vocab_size": 152064 + }, + "user_token_id": 872, + "video_token_id": 151656, + "vision_config": { + "_name_or_path": "", + "add_cross_attention": false, + "apply_vit_abs_pos_embed": true, + "architectures": null, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "deepstack_visual_indexes": [ + 8, + 16, + 24 + ], + "depth": 27, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 768, + "in_channels": 3, + "in_chans": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "model_type": "qwen3_omni_moe_vision_encoder", + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_heads": 16, + "num_position_embeddings": 2304, + "num_return_sequences": 1, + "out_hidden_size": 2048, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 16, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "spatial_merge_size": 2, + "spatial_patch_size": 16, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "temporal_patch_size": 2, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "tokens_per_second": 2, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 + }, + "transformers_version": "4.57.3", + "tts_bos_token_id": 151672, + "tts_eos_token_id": 151673, + "tts_pad_token_id": 151671, + "user_token_id": 872 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/iter_0016000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/iter_0016000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..0203eef2f2576370c91e6dbca27e344aa960d5b0 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/iter_0016000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:565776706cfcaa68ae90ab6d1dd2ae983db3803fd8ca951bd0d5f4882d0ef845 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..e0a264e3b1b9a105a5389617b1ccff64fe1bc6ea --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +16000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7df5b45379044a680218cb2b7ed4d01a6633f537 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/preprocessor_config.json @@ -0,0 +1,30 @@ +{ + "dither": 0.0, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_pixels": 12845056, + "merge_size": 2, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "return_attention_mask": true, + "sampling_rate": 16000, + "temporal_patch_size": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/special_tokens_map.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..fca9909b535ff91b68474e5ee7258e1574e60d1e --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/special_tokens_map.json @@ -0,0 +1,44 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "image_token": "<|image_pad|>", + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/tokenizer_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1491f11cc7fc66a763bcda6f4f6b8c3d026845 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/tokenizer_config.json @@ -0,0 +1,317 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151669": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151670": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151675": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "image_token": "<|image_pad|>", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3OmniMoeProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/video_preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6e0fb77bf83924807f06b6f314210795f76c2e42 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000-merged/video_preprocessor_config.json @@ -0,0 +1,54 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "dither": 0.0, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": false, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "fps": null, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "max_pixels": 12845056, + "merge_size": 2, + "min_frames": 4, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "num_frames": null, + "pad_size": null, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_attention_mask": true, + "return_metadata": false, + "sampling_rate": 16000, + "size": { + "longest_edge": 12845056, + "shortest_edge": 3136 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen2VLVideoProcessor" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000/iter_0016000/.metadata b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000/iter_0016000/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..88cb30525d6b0218d6eee4d02eb29ff3d289ae92 Binary files /dev/null and b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000/iter_0016000/.metadata differ diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000/iter_0016000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000/iter_0016000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..0203eef2f2576370c91e6dbca27e344aa960d5b0 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000/iter_0016000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:565776706cfcaa68ae90ab6d1dd2ae983db3803fd8ca951bd0d5f4882d0ef845 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000/iter_0016000/metadata.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000/iter_0016000/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000/iter_0016000/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..e0a264e3b1b9a105a5389617b1ccff64fe1bc6ea --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-16000/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +16000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/added_tokens.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..c962f0ea74c2dc3172f28f532b632288bf7d56c3 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/added_tokens.json @@ -0,0 +1,35 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "": 151671, + "": 151672, + "": 151674, + "": 151673, + "<|audio_end|>": 151670, + "<|audio_pad|>": 151675, + "<|audio_start|>": 151669, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/chat_template.jinja b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1efcce1402ae5d1d61d4a4cf56accea5921251cf --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/chat_template.jinja @@ -0,0 +1,122 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- "<|vision_start|><|image_pad|><|vision_end|>" }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- "<|audio_start|><|audio_pad|><|audio_end|>" }} + {%- elif content.type == 'video' or 'video' in content %} + {{- "<|vision_start|><|video_pad|><|vision_end|>" }} + {%- elif content.type == 'text' %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} + {{- '\n\n' }} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|image_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- '<|im_start|>system\n' +"<|audio_start|><|audio_pad|><|audio_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'video' or 'video' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|video_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'text' %} + {{- '<|im_start|>system\n' +content.text+ '<|im_end|>\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = namespace(text="") %} + {%- for mcontent in message.content %} + {%- if mcontent.type == 'image' or 'image' in mcontent or 'image_url' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|image_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'audio' or 'audio' in mcontent or 'audio_url' in mcontent %} + {%- set content.text = content.text~"<|audio_start|><|audio_pad|><|audio_end|>" %} + {%- elif mcontent.type == 'video' or 'video' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|video_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'text' %} + {%- set content.text = content.text~mcontent.text %} + {%- endif %} + {%- endfor %} + {%- set content = content.text %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = "" %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip("\n") + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %}{{- '\n' }}{%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}{{- '<|im_start|>user' }}{%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}{{- '<|im_end|>\n' }}{%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %}{{- '\n\n\n\n' }}{%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6dce3d213c49a2ea04a5bc9ba76b53e440cab76c --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/config.json @@ -0,0 +1,552 @@ +{ + "architectures": [ + "Qwen3OmniMoeForConditionalGeneration" + ], + "assistant_token_id": 77091, + "code2wav_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "codebook_dim": 512, + "codebook_size": 2048, + "decoder_dim": 1536, + "hidden_act": "silu", + "hidden_size": 1024, + "intermediate_size": 3072, + "layer_scale_initial_scale": 0.01, + "max_position_embeddings": 8000, + "model_type": "", + "num_attention_heads": 16, + "num_hidden_layers": 8, + "num_key_value_heads": 16, + "num_quantizers": 16, + "num_semantic_quantizers": 1, + "rms_norm_eps": 1e-05, + "rope_theta": 10000, + "semantic_codebook_size": 4096, + "sliding_window": 72, + "upsample_rates": [ + 8, + 5, + 4, + 3 + ], + "upsampling_ratios": [ + 2, + 2 + ], + "vector_quantization_hidden_dimension": 512 + }, + "dtype": "bfloat16", + "enable_audio_output": true, + "hidden_size": 1024, + "im_end_token_id": 151645, + "im_start_token_id": 151644, + "keys_to_ignore_at_inference": [ + "past_key_values" + ], + "model_type": "qwen3_omni_moe", + "system_token_id": 8948, + "talker_config": { + "accept_hidden_layer": 24, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "code_predictor_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": null, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 3072, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 32768, + "max_window_layers": 28, + "min_length": 0, + "model_type": "qwen3_omni_moe_talker_code_predictor", + "no_repeat_ngram_size": 0, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_code_groups": 16, + "num_hidden_layers": 5, + "num_key_value_heads": 8, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sep_token_id": null, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 2048 + }, + "codec_bos_id": 2149, + "codec_eos_token_id": 2150, + "codec_nothink_id": 2155, + "codec_pad_id": 2148, + "codec_think_bos_id": 2156, + "codec_think_eos_id": 2157, + "image_token_id": 151655, + "model_type": "", + "num_code_groups": 16, + "output_router_logits": false, + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "spatial_merge_size": 2, + "speaker_id": { + "aiden": 2303, + "chelsie": 2301, + "ethan": 2302 + }, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 2048, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_talker_text", + "moe_intermediate_size": 384, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 6, + "num_hidden_layers": 20, + "num_key_value_heads": 2, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 768, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 3072 + }, + "thinker_hidden_size": 2048, + "video_token_id": 151656, + "vision_start_token_id": 151652 + }, + "thinker_config": { + "audio_config": { + "_name_or_path": "", + "activation_dropout": 0, + "activation_function": "gelu", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "conv_chunksize": 500, + "cross_attention_hidden_size": null, + "d_model": 1280, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "downsample_hidden_size": 480, + "dropout": 0, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layers": 32, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_source_positions": 1500, + "min_length": 0, + "model_type": "qwen3_omni_moe_audio_encoder", + "n_window": 50, + "n_window_infer": 800, + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 32, + "num_mel_bins": 128, + "num_return_sequences": 1, + "output_attentions": false, + "output_dim": 2048, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "scale_embedding": false, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "dtype": "bfloat16", + "image_token_id": 151655, + "initializer_range": 0.02, + "model_type": "qwen3_omni_moe_thinker", + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0.0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 768, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_text", + "moe_intermediate_size": 768, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 32, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 48, + "num_key_value_heads": 4, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 0, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_qk_norm": true, + "use_sliding_window": false, + "vocab_size": 152064 + }, + "user_token_id": 872, + "video_token_id": 151656, + "vision_config": { + "_name_or_path": "", + "add_cross_attention": false, + "apply_vit_abs_pos_embed": true, + "architectures": null, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "deepstack_visual_indexes": [ + 8, + 16, + 24 + ], + "depth": 27, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 768, + "in_channels": 3, + "in_chans": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "model_type": "qwen3_omni_moe_vision_encoder", + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_heads": 16, + "num_position_embeddings": 2304, + "num_return_sequences": 1, + "out_hidden_size": 2048, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 16, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "spatial_merge_size": 2, + "spatial_patch_size": 16, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "temporal_patch_size": 2, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "tokens_per_second": 2, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 + }, + "transformers_version": "4.57.3", + "tts_bos_token_id": 151672, + "tts_eos_token_id": 151673, + "tts_pad_token_id": 151671, + "user_token_id": 872 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/iter_0017000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/iter_0017000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3a59e9fcee421f351bc23a39219c70577bc3217 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/iter_0017000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0509f1c397a83016aeef4c4d89d170b2ad84f37ec80c3205bcd201cd3512b8d0 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..029d1a32ff28f2d499013b223e1097176d99cfcd --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +17000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7df5b45379044a680218cb2b7ed4d01a6633f537 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/preprocessor_config.json @@ -0,0 +1,30 @@ +{ + "dither": 0.0, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_pixels": 12845056, + "merge_size": 2, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "return_attention_mask": true, + "sampling_rate": 16000, + "temporal_patch_size": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/special_tokens_map.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..fca9909b535ff91b68474e5ee7258e1574e60d1e --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/special_tokens_map.json @@ -0,0 +1,44 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "image_token": "<|image_pad|>", + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/tokenizer_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1491f11cc7fc66a763bcda6f4f6b8c3d026845 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/tokenizer_config.json @@ -0,0 +1,317 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151669": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151670": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151675": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "image_token": "<|image_pad|>", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3OmniMoeProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/video_preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6e0fb77bf83924807f06b6f314210795f76c2e42 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000-merged/video_preprocessor_config.json @@ -0,0 +1,54 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "dither": 0.0, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": false, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "fps": null, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "max_pixels": 12845056, + "merge_size": 2, + "min_frames": 4, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "num_frames": null, + "pad_size": null, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_attention_mask": true, + "return_metadata": false, + "sampling_rate": 16000, + "size": { + "longest_edge": 12845056, + "shortest_edge": 3136 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen2VLVideoProcessor" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000/iter_0017000/.metadata b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000/iter_0017000/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..f0043be8f28e0c4636208043c7e43c6ebe51f705 Binary files /dev/null and b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000/iter_0017000/.metadata differ diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000/iter_0017000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000/iter_0017000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3a59e9fcee421f351bc23a39219c70577bc3217 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000/iter_0017000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0509f1c397a83016aeef4c4d89d170b2ad84f37ec80c3205bcd201cd3512b8d0 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000/iter_0017000/metadata.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000/iter_0017000/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000/iter_0017000/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..029d1a32ff28f2d499013b223e1097176d99cfcd --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-17000/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +17000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/added_tokens.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..c962f0ea74c2dc3172f28f532b632288bf7d56c3 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/added_tokens.json @@ -0,0 +1,35 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "": 151671, + "": 151672, + "": 151674, + "": 151673, + "<|audio_end|>": 151670, + "<|audio_pad|>": 151675, + "<|audio_start|>": 151669, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/chat_template.jinja b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1efcce1402ae5d1d61d4a4cf56accea5921251cf --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/chat_template.jinja @@ -0,0 +1,122 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- "<|vision_start|><|image_pad|><|vision_end|>" }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- "<|audio_start|><|audio_pad|><|audio_end|>" }} + {%- elif content.type == 'video' or 'video' in content %} + {{- "<|vision_start|><|video_pad|><|vision_end|>" }} + {%- elif content.type == 'text' %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} + {{- '\n\n' }} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|image_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- '<|im_start|>system\n' +"<|audio_start|><|audio_pad|><|audio_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'video' or 'video' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|video_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'text' %} + {{- '<|im_start|>system\n' +content.text+ '<|im_end|>\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = namespace(text="") %} + {%- for mcontent in message.content %} + {%- if mcontent.type == 'image' or 'image' in mcontent or 'image_url' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|image_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'audio' or 'audio' in mcontent or 'audio_url' in mcontent %} + {%- set content.text = content.text~"<|audio_start|><|audio_pad|><|audio_end|>" %} + {%- elif mcontent.type == 'video' or 'video' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|video_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'text' %} + {%- set content.text = content.text~mcontent.text %} + {%- endif %} + {%- endfor %} + {%- set content = content.text %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = "" %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip("\n") + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %}{{- '\n' }}{%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}{{- '<|im_start|>user' }}{%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}{{- '<|im_end|>\n' }}{%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %}{{- '\n\n\n\n' }}{%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6dce3d213c49a2ea04a5bc9ba76b53e440cab76c --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/config.json @@ -0,0 +1,552 @@ +{ + "architectures": [ + "Qwen3OmniMoeForConditionalGeneration" + ], + "assistant_token_id": 77091, + "code2wav_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "codebook_dim": 512, + "codebook_size": 2048, + "decoder_dim": 1536, + "hidden_act": "silu", + "hidden_size": 1024, + "intermediate_size": 3072, + "layer_scale_initial_scale": 0.01, + "max_position_embeddings": 8000, + "model_type": "", + "num_attention_heads": 16, + "num_hidden_layers": 8, + "num_key_value_heads": 16, + "num_quantizers": 16, + "num_semantic_quantizers": 1, + "rms_norm_eps": 1e-05, + "rope_theta": 10000, + "semantic_codebook_size": 4096, + "sliding_window": 72, + "upsample_rates": [ + 8, + 5, + 4, + 3 + ], + "upsampling_ratios": [ + 2, + 2 + ], + "vector_quantization_hidden_dimension": 512 + }, + "dtype": "bfloat16", + "enable_audio_output": true, + "hidden_size": 1024, + "im_end_token_id": 151645, + "im_start_token_id": 151644, + "keys_to_ignore_at_inference": [ + "past_key_values" + ], + "model_type": "qwen3_omni_moe", + "system_token_id": 8948, + "talker_config": { + "accept_hidden_layer": 24, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "code_predictor_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": null, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 3072, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 32768, + "max_window_layers": 28, + "min_length": 0, + "model_type": "qwen3_omni_moe_talker_code_predictor", + "no_repeat_ngram_size": 0, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_code_groups": 16, + "num_hidden_layers": 5, + "num_key_value_heads": 8, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sep_token_id": null, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 2048 + }, + "codec_bos_id": 2149, + "codec_eos_token_id": 2150, + "codec_nothink_id": 2155, + "codec_pad_id": 2148, + "codec_think_bos_id": 2156, + "codec_think_eos_id": 2157, + "image_token_id": 151655, + "model_type": "", + "num_code_groups": 16, + "output_router_logits": false, + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "spatial_merge_size": 2, + "speaker_id": { + "aiden": 2303, + "chelsie": 2301, + "ethan": 2302 + }, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 2048, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_talker_text", + "moe_intermediate_size": 384, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 6, + "num_hidden_layers": 20, + "num_key_value_heads": 2, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 768, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 3072 + }, + "thinker_hidden_size": 2048, + "video_token_id": 151656, + "vision_start_token_id": 151652 + }, + "thinker_config": { + "audio_config": { + "_name_or_path": "", + "activation_dropout": 0, + "activation_function": "gelu", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "conv_chunksize": 500, + "cross_attention_hidden_size": null, + "d_model": 1280, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "downsample_hidden_size": 480, + "dropout": 0, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layers": 32, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_source_positions": 1500, + "min_length": 0, + "model_type": "qwen3_omni_moe_audio_encoder", + "n_window": 50, + "n_window_infer": 800, + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 32, + "num_mel_bins": 128, + "num_return_sequences": 1, + "output_attentions": false, + "output_dim": 2048, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "scale_embedding": false, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "dtype": "bfloat16", + "image_token_id": 151655, + "initializer_range": 0.02, + "model_type": "qwen3_omni_moe_thinker", + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0.0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 768, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_text", + "moe_intermediate_size": 768, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 32, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 48, + "num_key_value_heads": 4, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 0, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_qk_norm": true, + "use_sliding_window": false, + "vocab_size": 152064 + }, + "user_token_id": 872, + "video_token_id": 151656, + "vision_config": { + "_name_or_path": "", + "add_cross_attention": false, + "apply_vit_abs_pos_embed": true, + "architectures": null, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "deepstack_visual_indexes": [ + 8, + 16, + 24 + ], + "depth": 27, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 768, + "in_channels": 3, + "in_chans": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "model_type": "qwen3_omni_moe_vision_encoder", + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_heads": 16, + "num_position_embeddings": 2304, + "num_return_sequences": 1, + "out_hidden_size": 2048, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 16, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "spatial_merge_size": 2, + "spatial_patch_size": 16, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "temporal_patch_size": 2, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "tokens_per_second": 2, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 + }, + "transformers_version": "4.57.3", + "tts_bos_token_id": 151672, + "tts_eos_token_id": 151673, + "tts_pad_token_id": 151671, + "user_token_id": 872 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/iter_0018000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/iter_0018000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..e07552d6ada66b267c20cd52a7655ded2e358da2 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/iter_0018000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:761b55b0457567953e1edcfe211102b9bf9409c287e4bcf0af75da97bfb9c4fe +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..53dfdd5b2ae79e7447f1eb3879e262bee46f3473 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +18000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7df5b45379044a680218cb2b7ed4d01a6633f537 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/preprocessor_config.json @@ -0,0 +1,30 @@ +{ + "dither": 0.0, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_pixels": 12845056, + "merge_size": 2, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "return_attention_mask": true, + "sampling_rate": 16000, + "temporal_patch_size": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/special_tokens_map.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..fca9909b535ff91b68474e5ee7258e1574e60d1e --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/special_tokens_map.json @@ -0,0 +1,44 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "image_token": "<|image_pad|>", + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/tokenizer_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1491f11cc7fc66a763bcda6f4f6b8c3d026845 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/tokenizer_config.json @@ -0,0 +1,317 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151669": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151670": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151675": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "image_token": "<|image_pad|>", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3OmniMoeProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/video_preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6e0fb77bf83924807f06b6f314210795f76c2e42 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000-merged/video_preprocessor_config.json @@ -0,0 +1,54 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "dither": 0.0, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": false, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "fps": null, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "max_pixels": 12845056, + "merge_size": 2, + "min_frames": 4, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "num_frames": null, + "pad_size": null, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_attention_mask": true, + "return_metadata": false, + "sampling_rate": 16000, + "size": { + "longest_edge": 12845056, + "shortest_edge": 3136 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen2VLVideoProcessor" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000/iter_0018000/.metadata b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000/iter_0018000/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..e1d7b57b3e9ef56346bc0a04b873e027e54216fe Binary files /dev/null and b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000/iter_0018000/.metadata differ diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000/iter_0018000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000/iter_0018000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..e07552d6ada66b267c20cd52a7655ded2e358da2 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000/iter_0018000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:761b55b0457567953e1edcfe211102b9bf9409c287e4bcf0af75da97bfb9c4fe +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000/iter_0018000/metadata.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000/iter_0018000/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000/iter_0018000/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..53dfdd5b2ae79e7447f1eb3879e262bee46f3473 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-18000/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +18000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/added_tokens.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..c962f0ea74c2dc3172f28f532b632288bf7d56c3 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/added_tokens.json @@ -0,0 +1,35 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "": 151671, + "": 151672, + "": 151674, + "": 151673, + "<|audio_end|>": 151670, + "<|audio_pad|>": 151675, + "<|audio_start|>": 151669, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/chat_template.jinja b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1efcce1402ae5d1d61d4a4cf56accea5921251cf --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/chat_template.jinja @@ -0,0 +1,122 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- "<|vision_start|><|image_pad|><|vision_end|>" }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- "<|audio_start|><|audio_pad|><|audio_end|>" }} + {%- elif content.type == 'video' or 'video' in content %} + {{- "<|vision_start|><|video_pad|><|vision_end|>" }} + {%- elif content.type == 'text' %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} + {{- '\n\n' }} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|image_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- '<|im_start|>system\n' +"<|audio_start|><|audio_pad|><|audio_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'video' or 'video' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|video_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'text' %} + {{- '<|im_start|>system\n' +content.text+ '<|im_end|>\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = namespace(text="") %} + {%- for mcontent in message.content %} + {%- if mcontent.type == 'image' or 'image' in mcontent or 'image_url' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|image_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'audio' or 'audio' in mcontent or 'audio_url' in mcontent %} + {%- set content.text = content.text~"<|audio_start|><|audio_pad|><|audio_end|>" %} + {%- elif mcontent.type == 'video' or 'video' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|video_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'text' %} + {%- set content.text = content.text~mcontent.text %} + {%- endif %} + {%- endfor %} + {%- set content = content.text %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = "" %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip("\n") + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %}{{- '\n' }}{%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}{{- '<|im_start|>user' }}{%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}{{- '<|im_end|>\n' }}{%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %}{{- '\n\n\n\n' }}{%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6dce3d213c49a2ea04a5bc9ba76b53e440cab76c --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/config.json @@ -0,0 +1,552 @@ +{ + "architectures": [ + "Qwen3OmniMoeForConditionalGeneration" + ], + "assistant_token_id": 77091, + "code2wav_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "codebook_dim": 512, + "codebook_size": 2048, + "decoder_dim": 1536, + "hidden_act": "silu", + "hidden_size": 1024, + "intermediate_size": 3072, + "layer_scale_initial_scale": 0.01, + "max_position_embeddings": 8000, + "model_type": "", + "num_attention_heads": 16, + "num_hidden_layers": 8, + "num_key_value_heads": 16, + "num_quantizers": 16, + "num_semantic_quantizers": 1, + "rms_norm_eps": 1e-05, + "rope_theta": 10000, + "semantic_codebook_size": 4096, + "sliding_window": 72, + "upsample_rates": [ + 8, + 5, + 4, + 3 + ], + "upsampling_ratios": [ + 2, + 2 + ], + "vector_quantization_hidden_dimension": 512 + }, + "dtype": "bfloat16", + "enable_audio_output": true, + "hidden_size": 1024, + "im_end_token_id": 151645, + "im_start_token_id": 151644, + "keys_to_ignore_at_inference": [ + "past_key_values" + ], + "model_type": "qwen3_omni_moe", + "system_token_id": 8948, + "talker_config": { + "accept_hidden_layer": 24, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "code_predictor_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": null, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 3072, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 32768, + "max_window_layers": 28, + "min_length": 0, + "model_type": "qwen3_omni_moe_talker_code_predictor", + "no_repeat_ngram_size": 0, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_code_groups": 16, + "num_hidden_layers": 5, + "num_key_value_heads": 8, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sep_token_id": null, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 2048 + }, + "codec_bos_id": 2149, + "codec_eos_token_id": 2150, + "codec_nothink_id": 2155, + "codec_pad_id": 2148, + "codec_think_bos_id": 2156, + "codec_think_eos_id": 2157, + "image_token_id": 151655, + "model_type": "", + "num_code_groups": 16, + "output_router_logits": false, + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "spatial_merge_size": 2, + "speaker_id": { + "aiden": 2303, + "chelsie": 2301, + "ethan": 2302 + }, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 2048, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_talker_text", + "moe_intermediate_size": 384, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 6, + "num_hidden_layers": 20, + "num_key_value_heads": 2, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 768, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 3072 + }, + "thinker_hidden_size": 2048, + "video_token_id": 151656, + "vision_start_token_id": 151652 + }, + "thinker_config": { + "audio_config": { + "_name_or_path": "", + "activation_dropout": 0, + "activation_function": "gelu", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "conv_chunksize": 500, + "cross_attention_hidden_size": null, + "d_model": 1280, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "downsample_hidden_size": 480, + "dropout": 0, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layers": 32, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_source_positions": 1500, + "min_length": 0, + "model_type": "qwen3_omni_moe_audio_encoder", + "n_window": 50, + "n_window_infer": 800, + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 32, + "num_mel_bins": 128, + "num_return_sequences": 1, + "output_attentions": false, + "output_dim": 2048, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "scale_embedding": false, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "dtype": "bfloat16", + "image_token_id": 151655, + "initializer_range": 0.02, + "model_type": "qwen3_omni_moe_thinker", + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0.0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 768, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_text", + "moe_intermediate_size": 768, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 32, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 48, + "num_key_value_heads": 4, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 0, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_qk_norm": true, + "use_sliding_window": false, + "vocab_size": 152064 + }, + "user_token_id": 872, + "video_token_id": 151656, + "vision_config": { + "_name_or_path": "", + "add_cross_attention": false, + "apply_vit_abs_pos_embed": true, + "architectures": null, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "deepstack_visual_indexes": [ + 8, + 16, + 24 + ], + "depth": 27, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 768, + "in_channels": 3, + "in_chans": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "model_type": "qwen3_omni_moe_vision_encoder", + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_heads": 16, + "num_position_embeddings": 2304, + "num_return_sequences": 1, + "out_hidden_size": 2048, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 16, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "spatial_merge_size": 2, + "spatial_patch_size": 16, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "temporal_patch_size": 2, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "tokens_per_second": 2, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 + }, + "transformers_version": "4.57.3", + "tts_bos_token_id": 151672, + "tts_eos_token_id": 151673, + "tts_pad_token_id": 151671, + "user_token_id": 872 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/iter_0019000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/iter_0019000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c3efae2c467027dcfb102d9bee6531e51955aff --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/iter_0019000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:295149ed4dc4cbd37e364d5a6b76811853daa8668bdc18cb208a62e7aeb86f78 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e46fa036595fcfa752471203a4b02019965b544 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +19000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7df5b45379044a680218cb2b7ed4d01a6633f537 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/preprocessor_config.json @@ -0,0 +1,30 @@ +{ + "dither": 0.0, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_pixels": 12845056, + "merge_size": 2, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "return_attention_mask": true, + "sampling_rate": 16000, + "temporal_patch_size": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/special_tokens_map.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..fca9909b535ff91b68474e5ee7258e1574e60d1e --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/special_tokens_map.json @@ -0,0 +1,44 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "image_token": "<|image_pad|>", + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/tokenizer_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1491f11cc7fc66a763bcda6f4f6b8c3d026845 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/tokenizer_config.json @@ -0,0 +1,317 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151669": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151670": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151675": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "image_token": "<|image_pad|>", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3OmniMoeProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/video_preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6e0fb77bf83924807f06b6f314210795f76c2e42 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000-merged/video_preprocessor_config.json @@ -0,0 +1,54 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "dither": 0.0, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": false, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "fps": null, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "max_pixels": 12845056, + "merge_size": 2, + "min_frames": 4, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "num_frames": null, + "pad_size": null, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_attention_mask": true, + "return_metadata": false, + "sampling_rate": 16000, + "size": { + "longest_edge": 12845056, + "shortest_edge": 3136 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen2VLVideoProcessor" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000/iter_0019000/.metadata b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000/iter_0019000/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..dfb9d5041a0b4802aff84b694c2cbc6993179636 Binary files /dev/null and b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000/iter_0019000/.metadata differ diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000/iter_0019000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000/iter_0019000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c3efae2c467027dcfb102d9bee6531e51955aff --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000/iter_0019000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:295149ed4dc4cbd37e364d5a6b76811853daa8668bdc18cb208a62e7aeb86f78 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000/iter_0019000/metadata.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000/iter_0019000/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000/iter_0019000/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e46fa036595fcfa752471203a4b02019965b544 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-19000/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +19000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/added_tokens.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..c962f0ea74c2dc3172f28f532b632288bf7d56c3 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/added_tokens.json @@ -0,0 +1,35 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "": 151671, + "": 151672, + "": 151674, + "": 151673, + "<|audio_end|>": 151670, + "<|audio_pad|>": 151675, + "<|audio_start|>": 151669, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/chat_template.jinja b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1efcce1402ae5d1d61d4a4cf56accea5921251cf --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/chat_template.jinja @@ -0,0 +1,122 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- "<|vision_start|><|image_pad|><|vision_end|>" }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- "<|audio_start|><|audio_pad|><|audio_end|>" }} + {%- elif content.type == 'video' or 'video' in content %} + {{- "<|vision_start|><|video_pad|><|vision_end|>" }} + {%- elif content.type == 'text' %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} + {{- '\n\n' }} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|image_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- '<|im_start|>system\n' +"<|audio_start|><|audio_pad|><|audio_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'video' or 'video' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|video_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'text' %} + {{- '<|im_start|>system\n' +content.text+ '<|im_end|>\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = namespace(text="") %} + {%- for mcontent in message.content %} + {%- if mcontent.type == 'image' or 'image' in mcontent or 'image_url' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|image_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'audio' or 'audio' in mcontent or 'audio_url' in mcontent %} + {%- set content.text = content.text~"<|audio_start|><|audio_pad|><|audio_end|>" %} + {%- elif mcontent.type == 'video' or 'video' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|video_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'text' %} + {%- set content.text = content.text~mcontent.text %} + {%- endif %} + {%- endfor %} + {%- set content = content.text %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = "" %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip("\n") + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %}{{- '\n' }}{%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}{{- '<|im_start|>user' }}{%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}{{- '<|im_end|>\n' }}{%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %}{{- '\n\n\n\n' }}{%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6dce3d213c49a2ea04a5bc9ba76b53e440cab76c --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/config.json @@ -0,0 +1,552 @@ +{ + "architectures": [ + "Qwen3OmniMoeForConditionalGeneration" + ], + "assistant_token_id": 77091, + "code2wav_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "codebook_dim": 512, + "codebook_size": 2048, + "decoder_dim": 1536, + "hidden_act": "silu", + "hidden_size": 1024, + "intermediate_size": 3072, + "layer_scale_initial_scale": 0.01, + "max_position_embeddings": 8000, + "model_type": "", + "num_attention_heads": 16, + "num_hidden_layers": 8, + "num_key_value_heads": 16, + "num_quantizers": 16, + "num_semantic_quantizers": 1, + "rms_norm_eps": 1e-05, + "rope_theta": 10000, + "semantic_codebook_size": 4096, + "sliding_window": 72, + "upsample_rates": [ + 8, + 5, + 4, + 3 + ], + "upsampling_ratios": [ + 2, + 2 + ], + "vector_quantization_hidden_dimension": 512 + }, + "dtype": "bfloat16", + "enable_audio_output": true, + "hidden_size": 1024, + "im_end_token_id": 151645, + "im_start_token_id": 151644, + "keys_to_ignore_at_inference": [ + "past_key_values" + ], + "model_type": "qwen3_omni_moe", + "system_token_id": 8948, + "talker_config": { + "accept_hidden_layer": 24, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "code_predictor_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": null, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 3072, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 32768, + "max_window_layers": 28, + "min_length": 0, + "model_type": "qwen3_omni_moe_talker_code_predictor", + "no_repeat_ngram_size": 0, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_code_groups": 16, + "num_hidden_layers": 5, + "num_key_value_heads": 8, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sep_token_id": null, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 2048 + }, + "codec_bos_id": 2149, + "codec_eos_token_id": 2150, + "codec_nothink_id": 2155, + "codec_pad_id": 2148, + "codec_think_bos_id": 2156, + "codec_think_eos_id": 2157, + "image_token_id": 151655, + "model_type": "", + "num_code_groups": 16, + "output_router_logits": false, + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "spatial_merge_size": 2, + "speaker_id": { + "aiden": 2303, + "chelsie": 2301, + "ethan": 2302 + }, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 2048, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_talker_text", + "moe_intermediate_size": 384, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 6, + "num_hidden_layers": 20, + "num_key_value_heads": 2, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 768, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 3072 + }, + "thinker_hidden_size": 2048, + "video_token_id": 151656, + "vision_start_token_id": 151652 + }, + "thinker_config": { + "audio_config": { + "_name_or_path": "", + "activation_dropout": 0, + "activation_function": "gelu", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "conv_chunksize": 500, + "cross_attention_hidden_size": null, + "d_model": 1280, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "downsample_hidden_size": 480, + "dropout": 0, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layers": 32, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_source_positions": 1500, + "min_length": 0, + "model_type": "qwen3_omni_moe_audio_encoder", + "n_window": 50, + "n_window_infer": 800, + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 32, + "num_mel_bins": 128, + "num_return_sequences": 1, + "output_attentions": false, + "output_dim": 2048, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "scale_embedding": false, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "dtype": "bfloat16", + "image_token_id": 151655, + "initializer_range": 0.02, + "model_type": "qwen3_omni_moe_thinker", + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0.0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 768, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_text", + "moe_intermediate_size": 768, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 32, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 48, + "num_key_value_heads": 4, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 0, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_qk_norm": true, + "use_sliding_window": false, + "vocab_size": 152064 + }, + "user_token_id": 872, + "video_token_id": 151656, + "vision_config": { + "_name_or_path": "", + "add_cross_attention": false, + "apply_vit_abs_pos_embed": true, + "architectures": null, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "deepstack_visual_indexes": [ + 8, + 16, + 24 + ], + "depth": 27, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 768, + "in_channels": 3, + "in_chans": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "model_type": "qwen3_omni_moe_vision_encoder", + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_heads": 16, + "num_position_embeddings": 2304, + "num_return_sequences": 1, + "out_hidden_size": 2048, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 16, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "spatial_merge_size": 2, + "spatial_patch_size": 16, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "temporal_patch_size": 2, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "tokens_per_second": 2, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 + }, + "transformers_version": "4.57.3", + "tts_bos_token_id": 151672, + "tts_eos_token_id": 151673, + "tts_pad_token_id": 151671, + "user_token_id": 872 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/iter_0002000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/iter_0002000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..95598f45d0fa00266a05c46d4d3907304709044b --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/iter_0002000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dbfa150ee766980062e7c45ee077e124f64903c76f68e593707a22769818ca3 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..9463411b62f21b7ed88bbe711c958b3b66153330 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +2000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7df5b45379044a680218cb2b7ed4d01a6633f537 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/preprocessor_config.json @@ -0,0 +1,30 @@ +{ + "dither": 0.0, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_pixels": 12845056, + "merge_size": 2, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "return_attention_mask": true, + "sampling_rate": 16000, + "temporal_patch_size": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/special_tokens_map.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..fca9909b535ff91b68474e5ee7258e1574e60d1e --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/special_tokens_map.json @@ -0,0 +1,44 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "image_token": "<|image_pad|>", + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/tokenizer_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1491f11cc7fc66a763bcda6f4f6b8c3d026845 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/tokenizer_config.json @@ -0,0 +1,317 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151669": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151670": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151675": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "image_token": "<|image_pad|>", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3OmniMoeProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/video_preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6e0fb77bf83924807f06b6f314210795f76c2e42 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/video_preprocessor_config.json @@ -0,0 +1,54 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "dither": 0.0, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": false, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "fps": null, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "max_pixels": 12845056, + "merge_size": 2, + "min_frames": 4, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "num_frames": null, + "pad_size": null, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_attention_mask": true, + "return_metadata": false, + "sampling_rate": 16000, + "size": { + "longest_edge": 12845056, + "shortest_edge": 3136 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen2VLVideoProcessor" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/.metadata b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..8d9f0c5faa6a26a66be62490a72a687e4a0732d2 Binary files /dev/null and b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/.metadata differ diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..95598f45d0fa00266a05c46d4d3907304709044b --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dbfa150ee766980062e7c45ee077e124f64903c76f68e593707a22769818ca3 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/metadata.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..9463411b62f21b7ed88bbe711c958b3b66153330 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +2000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/added_tokens.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..c962f0ea74c2dc3172f28f532b632288bf7d56c3 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/added_tokens.json @@ -0,0 +1,35 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "": 151671, + "": 151672, + "": 151674, + "": 151673, + "<|audio_end|>": 151670, + "<|audio_pad|>": 151675, + "<|audio_start|>": 151669, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/chat_template.jinja b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1efcce1402ae5d1d61d4a4cf56accea5921251cf --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/chat_template.jinja @@ -0,0 +1,122 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- "<|vision_start|><|image_pad|><|vision_end|>" }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- "<|audio_start|><|audio_pad|><|audio_end|>" }} + {%- elif content.type == 'video' or 'video' in content %} + {{- "<|vision_start|><|video_pad|><|vision_end|>" }} + {%- elif content.type == 'text' %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} + {{- '\n\n' }} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|image_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- '<|im_start|>system\n' +"<|audio_start|><|audio_pad|><|audio_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'video' or 'video' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|video_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'text' %} + {{- '<|im_start|>system\n' +content.text+ '<|im_end|>\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = namespace(text="") %} + {%- for mcontent in message.content %} + {%- if mcontent.type == 'image' or 'image' in mcontent or 'image_url' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|image_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'audio' or 'audio' in mcontent or 'audio_url' in mcontent %} + {%- set content.text = content.text~"<|audio_start|><|audio_pad|><|audio_end|>" %} + {%- elif mcontent.type == 'video' or 'video' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|video_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'text' %} + {%- set content.text = content.text~mcontent.text %} + {%- endif %} + {%- endfor %} + {%- set content = content.text %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = "" %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip("\n") + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %}{{- '\n' }}{%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}{{- '<|im_start|>user' }}{%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}{{- '<|im_end|>\n' }}{%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %}{{- '\n\n\n\n' }}{%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6dce3d213c49a2ea04a5bc9ba76b53e440cab76c --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/config.json @@ -0,0 +1,552 @@ +{ + "architectures": [ + "Qwen3OmniMoeForConditionalGeneration" + ], + "assistant_token_id": 77091, + "code2wav_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "codebook_dim": 512, + "codebook_size": 2048, + "decoder_dim": 1536, + "hidden_act": "silu", + "hidden_size": 1024, + "intermediate_size": 3072, + "layer_scale_initial_scale": 0.01, + "max_position_embeddings": 8000, + "model_type": "", + "num_attention_heads": 16, + "num_hidden_layers": 8, + "num_key_value_heads": 16, + "num_quantizers": 16, + "num_semantic_quantizers": 1, + "rms_norm_eps": 1e-05, + "rope_theta": 10000, + "semantic_codebook_size": 4096, + "sliding_window": 72, + "upsample_rates": [ + 8, + 5, + 4, + 3 + ], + "upsampling_ratios": [ + 2, + 2 + ], + "vector_quantization_hidden_dimension": 512 + }, + "dtype": "bfloat16", + "enable_audio_output": true, + "hidden_size": 1024, + "im_end_token_id": 151645, + "im_start_token_id": 151644, + "keys_to_ignore_at_inference": [ + "past_key_values" + ], + "model_type": "qwen3_omni_moe", + "system_token_id": 8948, + "talker_config": { + "accept_hidden_layer": 24, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "code_predictor_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": null, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 3072, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 32768, + "max_window_layers": 28, + "min_length": 0, + "model_type": "qwen3_omni_moe_talker_code_predictor", + "no_repeat_ngram_size": 0, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_code_groups": 16, + "num_hidden_layers": 5, + "num_key_value_heads": 8, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sep_token_id": null, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 2048 + }, + "codec_bos_id": 2149, + "codec_eos_token_id": 2150, + "codec_nothink_id": 2155, + "codec_pad_id": 2148, + "codec_think_bos_id": 2156, + "codec_think_eos_id": 2157, + "image_token_id": 151655, + "model_type": "", + "num_code_groups": 16, + "output_router_logits": false, + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "spatial_merge_size": 2, + "speaker_id": { + "aiden": 2303, + "chelsie": 2301, + "ethan": 2302 + }, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 2048, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_talker_text", + "moe_intermediate_size": 384, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 6, + "num_hidden_layers": 20, + "num_key_value_heads": 2, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 768, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 3072 + }, + "thinker_hidden_size": 2048, + "video_token_id": 151656, + "vision_start_token_id": 151652 + }, + "thinker_config": { + "audio_config": { + "_name_or_path": "", + "activation_dropout": 0, + "activation_function": "gelu", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "conv_chunksize": 500, + "cross_attention_hidden_size": null, + "d_model": 1280, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "downsample_hidden_size": 480, + "dropout": 0, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layers": 32, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_source_positions": 1500, + "min_length": 0, + "model_type": "qwen3_omni_moe_audio_encoder", + "n_window": 50, + "n_window_infer": 800, + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 32, + "num_mel_bins": 128, + "num_return_sequences": 1, + "output_attentions": false, + "output_dim": 2048, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "scale_embedding": false, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "dtype": "bfloat16", + "image_token_id": 151655, + "initializer_range": 0.02, + "model_type": "qwen3_omni_moe_thinker", + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0.0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 768, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_text", + "moe_intermediate_size": 768, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 32, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 48, + "num_key_value_heads": 4, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 0, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_qk_norm": true, + "use_sliding_window": false, + "vocab_size": 152064 + }, + "user_token_id": 872, + "video_token_id": 151656, + "vision_config": { + "_name_or_path": "", + "add_cross_attention": false, + "apply_vit_abs_pos_embed": true, + "architectures": null, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "deepstack_visual_indexes": [ + 8, + 16, + 24 + ], + "depth": 27, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 768, + "in_channels": 3, + "in_chans": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "model_type": "qwen3_omni_moe_vision_encoder", + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_heads": 16, + "num_position_embeddings": 2304, + "num_return_sequences": 1, + "out_hidden_size": 2048, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 16, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "spatial_merge_size": 2, + "spatial_patch_size": 16, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "temporal_patch_size": 2, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "tokens_per_second": 2, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 + }, + "transformers_version": "4.57.3", + "tts_bos_token_id": 151672, + "tts_eos_token_id": 151673, + "tts_pad_token_id": 151671, + "user_token_id": 872 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/iter_0020000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/iter_0020000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..6becf0806c4e26769acd2df1f2feb57dfae0972b --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/iter_0020000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28de2db36507703129293eef8394141991e45f3e28029a6062c54b29e2e31193 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..d0000aa1e3dab6f3e31403e2c8ddcf2bde82e00c --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +20000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7df5b45379044a680218cb2b7ed4d01a6633f537 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/preprocessor_config.json @@ -0,0 +1,30 @@ +{ + "dither": 0.0, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_pixels": 12845056, + "merge_size": 2, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "return_attention_mask": true, + "sampling_rate": 16000, + "temporal_patch_size": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/special_tokens_map.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..fca9909b535ff91b68474e5ee7258e1574e60d1e --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/special_tokens_map.json @@ -0,0 +1,44 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "image_token": "<|image_pad|>", + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/tokenizer_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1491f11cc7fc66a763bcda6f4f6b8c3d026845 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/tokenizer_config.json @@ -0,0 +1,317 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151669": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151670": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151675": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "image_token": "<|image_pad|>", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3OmniMoeProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/video_preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6e0fb77bf83924807f06b6f314210795f76c2e42 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000-merged/video_preprocessor_config.json @@ -0,0 +1,54 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "dither": 0.0, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": false, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "fps": null, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "max_pixels": 12845056, + "merge_size": 2, + "min_frames": 4, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "num_frames": null, + "pad_size": null, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_attention_mask": true, + "return_metadata": false, + "sampling_rate": 16000, + "size": { + "longest_edge": 12845056, + "shortest_edge": 3136 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen2VLVideoProcessor" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000/iter_0020000/.metadata b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000/iter_0020000/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..bcf3d6f4496212808395a6143be2044157f9b48e Binary files /dev/null and b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000/iter_0020000/.metadata differ diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000/iter_0020000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000/iter_0020000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..6becf0806c4e26769acd2df1f2feb57dfae0972b --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000/iter_0020000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28de2db36507703129293eef8394141991e45f3e28029a6062c54b29e2e31193 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000/iter_0020000/metadata.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000/iter_0020000/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000/iter_0020000/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..d0000aa1e3dab6f3e31403e2c8ddcf2bde82e00c --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-20000/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +20000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/added_tokens.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..c962f0ea74c2dc3172f28f532b632288bf7d56c3 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/added_tokens.json @@ -0,0 +1,35 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "": 151671, + "": 151672, + "": 151674, + "": 151673, + "<|audio_end|>": 151670, + "<|audio_pad|>": 151675, + "<|audio_start|>": 151669, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/chat_template.jinja b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1efcce1402ae5d1d61d4a4cf56accea5921251cf --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/chat_template.jinja @@ -0,0 +1,122 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- "<|vision_start|><|image_pad|><|vision_end|>" }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- "<|audio_start|><|audio_pad|><|audio_end|>" }} + {%- elif content.type == 'video' or 'video' in content %} + {{- "<|vision_start|><|video_pad|><|vision_end|>" }} + {%- elif content.type == 'text' %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} + {{- '\n\n' }} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|image_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- '<|im_start|>system\n' +"<|audio_start|><|audio_pad|><|audio_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'video' or 'video' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|video_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'text' %} + {{- '<|im_start|>system\n' +content.text+ '<|im_end|>\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = namespace(text="") %} + {%- for mcontent in message.content %} + {%- if mcontent.type == 'image' or 'image' in mcontent or 'image_url' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|image_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'audio' or 'audio' in mcontent or 'audio_url' in mcontent %} + {%- set content.text = content.text~"<|audio_start|><|audio_pad|><|audio_end|>" %} + {%- elif mcontent.type == 'video' or 'video' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|video_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'text' %} + {%- set content.text = content.text~mcontent.text %} + {%- endif %} + {%- endfor %} + {%- set content = content.text %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = "" %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip("\n") + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %}{{- '\n' }}{%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}{{- '<|im_start|>user' }}{%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}{{- '<|im_end|>\n' }}{%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %}{{- '\n\n\n\n' }}{%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6dce3d213c49a2ea04a5bc9ba76b53e440cab76c --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/config.json @@ -0,0 +1,552 @@ +{ + "architectures": [ + "Qwen3OmniMoeForConditionalGeneration" + ], + "assistant_token_id": 77091, + "code2wav_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "codebook_dim": 512, + "codebook_size": 2048, + "decoder_dim": 1536, + "hidden_act": "silu", + "hidden_size": 1024, + "intermediate_size": 3072, + "layer_scale_initial_scale": 0.01, + "max_position_embeddings": 8000, + "model_type": "", + "num_attention_heads": 16, + "num_hidden_layers": 8, + "num_key_value_heads": 16, + "num_quantizers": 16, + "num_semantic_quantizers": 1, + "rms_norm_eps": 1e-05, + "rope_theta": 10000, + "semantic_codebook_size": 4096, + "sliding_window": 72, + "upsample_rates": [ + 8, + 5, + 4, + 3 + ], + "upsampling_ratios": [ + 2, + 2 + ], + "vector_quantization_hidden_dimension": 512 + }, + "dtype": "bfloat16", + "enable_audio_output": true, + "hidden_size": 1024, + "im_end_token_id": 151645, + "im_start_token_id": 151644, + "keys_to_ignore_at_inference": [ + "past_key_values" + ], + "model_type": "qwen3_omni_moe", + "system_token_id": 8948, + "talker_config": { + "accept_hidden_layer": 24, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "code_predictor_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": null, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 3072, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 32768, + "max_window_layers": 28, + "min_length": 0, + "model_type": "qwen3_omni_moe_talker_code_predictor", + "no_repeat_ngram_size": 0, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_code_groups": 16, + "num_hidden_layers": 5, + "num_key_value_heads": 8, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sep_token_id": null, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 2048 + }, + "codec_bos_id": 2149, + "codec_eos_token_id": 2150, + "codec_nothink_id": 2155, + "codec_pad_id": 2148, + "codec_think_bos_id": 2156, + "codec_think_eos_id": 2157, + "image_token_id": 151655, + "model_type": "", + "num_code_groups": 16, + "output_router_logits": false, + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "spatial_merge_size": 2, + "speaker_id": { + "aiden": 2303, + "chelsie": 2301, + "ethan": 2302 + }, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 2048, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_talker_text", + "moe_intermediate_size": 384, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 6, + "num_hidden_layers": 20, + "num_key_value_heads": 2, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 768, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 3072 + }, + "thinker_hidden_size": 2048, + "video_token_id": 151656, + "vision_start_token_id": 151652 + }, + "thinker_config": { + "audio_config": { + "_name_or_path": "", + "activation_dropout": 0, + "activation_function": "gelu", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "conv_chunksize": 500, + "cross_attention_hidden_size": null, + "d_model": 1280, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "downsample_hidden_size": 480, + "dropout": 0, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layers": 32, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_source_positions": 1500, + "min_length": 0, + "model_type": "qwen3_omni_moe_audio_encoder", + "n_window": 50, + "n_window_infer": 800, + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 32, + "num_mel_bins": 128, + "num_return_sequences": 1, + "output_attentions": false, + "output_dim": 2048, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "scale_embedding": false, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "dtype": "bfloat16", + "image_token_id": 151655, + "initializer_range": 0.02, + "model_type": "qwen3_omni_moe_thinker", + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0.0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 768, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_text", + "moe_intermediate_size": 768, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 32, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 48, + "num_key_value_heads": 4, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 0, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_qk_norm": true, + "use_sliding_window": false, + "vocab_size": 152064 + }, + "user_token_id": 872, + "video_token_id": 151656, + "vision_config": { + "_name_or_path": "", + "add_cross_attention": false, + "apply_vit_abs_pos_embed": true, + "architectures": null, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "deepstack_visual_indexes": [ + 8, + 16, + 24 + ], + "depth": 27, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 768, + "in_channels": 3, + "in_chans": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "model_type": "qwen3_omni_moe_vision_encoder", + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_heads": 16, + "num_position_embeddings": 2304, + "num_return_sequences": 1, + "out_hidden_size": 2048, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 16, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "spatial_merge_size": 2, + "spatial_patch_size": 16, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "temporal_patch_size": 2, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "tokens_per_second": 2, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 + }, + "transformers_version": "4.57.3", + "tts_bos_token_id": 151672, + "tts_eos_token_id": 151673, + "tts_pad_token_id": 151671, + "user_token_id": 872 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/iter_0021000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/iter_0021000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..e43b152bf00f2c132342681d4f98d89cbe1c628c --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/iter_0021000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81781c3364cb7138f27f44a22db8bb3d900223315a0305b11835f02a5b534917 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..8eb46f452b747faa90362714f6a4b4957806feae --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +21000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7df5b45379044a680218cb2b7ed4d01a6633f537 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/preprocessor_config.json @@ -0,0 +1,30 @@ +{ + "dither": 0.0, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_pixels": 12845056, + "merge_size": 2, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "return_attention_mask": true, + "sampling_rate": 16000, + "temporal_patch_size": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/special_tokens_map.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..fca9909b535ff91b68474e5ee7258e1574e60d1e --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/special_tokens_map.json @@ -0,0 +1,44 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "image_token": "<|image_pad|>", + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/tokenizer_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1491f11cc7fc66a763bcda6f4f6b8c3d026845 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/tokenizer_config.json @@ -0,0 +1,317 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151669": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151670": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151675": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "image_token": "<|image_pad|>", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3OmniMoeProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/video_preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6e0fb77bf83924807f06b6f314210795f76c2e42 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000-merged/video_preprocessor_config.json @@ -0,0 +1,54 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "dither": 0.0, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": false, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "fps": null, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "max_pixels": 12845056, + "merge_size": 2, + "min_frames": 4, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "num_frames": null, + "pad_size": null, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_attention_mask": true, + "return_metadata": false, + "sampling_rate": 16000, + "size": { + "longest_edge": 12845056, + "shortest_edge": 3136 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen2VLVideoProcessor" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000/iter_0021000/.metadata b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000/iter_0021000/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..9673fdc99034ec240f99ee2e23c87f5b241b6d29 Binary files /dev/null and b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000/iter_0021000/.metadata differ diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000/iter_0021000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000/iter_0021000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..e43b152bf00f2c132342681d4f98d89cbe1c628c --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000/iter_0021000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81781c3364cb7138f27f44a22db8bb3d900223315a0305b11835f02a5b534917 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000/iter_0021000/metadata.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000/iter_0021000/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000/iter_0021000/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..8eb46f452b747faa90362714f6a4b4957806feae --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-21000/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +21000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/added_tokens.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..c962f0ea74c2dc3172f28f532b632288bf7d56c3 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/added_tokens.json @@ -0,0 +1,35 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "": 151671, + "": 151672, + "": 151674, + "": 151673, + "<|audio_end|>": 151670, + "<|audio_pad|>": 151675, + "<|audio_start|>": 151669, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/chat_template.jinja b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1efcce1402ae5d1d61d4a4cf56accea5921251cf --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/chat_template.jinja @@ -0,0 +1,122 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- "<|vision_start|><|image_pad|><|vision_end|>" }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- "<|audio_start|><|audio_pad|><|audio_end|>" }} + {%- elif content.type == 'video' or 'video' in content %} + {{- "<|vision_start|><|video_pad|><|vision_end|>" }} + {%- elif content.type == 'text' %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} + {{- '\n\n' }} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|image_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- '<|im_start|>system\n' +"<|audio_start|><|audio_pad|><|audio_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'video' or 'video' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|video_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'text' %} + {{- '<|im_start|>system\n' +content.text+ '<|im_end|>\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = namespace(text="") %} + {%- for mcontent in message.content %} + {%- if mcontent.type == 'image' or 'image' in mcontent or 'image_url' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|image_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'audio' or 'audio' in mcontent or 'audio_url' in mcontent %} + {%- set content.text = content.text~"<|audio_start|><|audio_pad|><|audio_end|>" %} + {%- elif mcontent.type == 'video' or 'video' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|video_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'text' %} + {%- set content.text = content.text~mcontent.text %} + {%- endif %} + {%- endfor %} + {%- set content = content.text %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = "" %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip("\n") + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %}{{- '\n' }}{%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}{{- '<|im_start|>user' }}{%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}{{- '<|im_end|>\n' }}{%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %}{{- '\n\n\n\n' }}{%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6dce3d213c49a2ea04a5bc9ba76b53e440cab76c --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/config.json @@ -0,0 +1,552 @@ +{ + "architectures": [ + "Qwen3OmniMoeForConditionalGeneration" + ], + "assistant_token_id": 77091, + "code2wav_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "codebook_dim": 512, + "codebook_size": 2048, + "decoder_dim": 1536, + "hidden_act": "silu", + "hidden_size": 1024, + "intermediate_size": 3072, + "layer_scale_initial_scale": 0.01, + "max_position_embeddings": 8000, + "model_type": "", + "num_attention_heads": 16, + "num_hidden_layers": 8, + "num_key_value_heads": 16, + "num_quantizers": 16, + "num_semantic_quantizers": 1, + "rms_norm_eps": 1e-05, + "rope_theta": 10000, + "semantic_codebook_size": 4096, + "sliding_window": 72, + "upsample_rates": [ + 8, + 5, + 4, + 3 + ], + "upsampling_ratios": [ + 2, + 2 + ], + "vector_quantization_hidden_dimension": 512 + }, + "dtype": "bfloat16", + "enable_audio_output": true, + "hidden_size": 1024, + "im_end_token_id": 151645, + "im_start_token_id": 151644, + "keys_to_ignore_at_inference": [ + "past_key_values" + ], + "model_type": "qwen3_omni_moe", + "system_token_id": 8948, + "talker_config": { + "accept_hidden_layer": 24, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "code_predictor_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": null, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 3072, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 32768, + "max_window_layers": 28, + "min_length": 0, + "model_type": "qwen3_omni_moe_talker_code_predictor", + "no_repeat_ngram_size": 0, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_code_groups": 16, + "num_hidden_layers": 5, + "num_key_value_heads": 8, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sep_token_id": null, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 2048 + }, + "codec_bos_id": 2149, + "codec_eos_token_id": 2150, + "codec_nothink_id": 2155, + "codec_pad_id": 2148, + "codec_think_bos_id": 2156, + "codec_think_eos_id": 2157, + "image_token_id": 151655, + "model_type": "", + "num_code_groups": 16, + "output_router_logits": false, + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "spatial_merge_size": 2, + "speaker_id": { + "aiden": 2303, + "chelsie": 2301, + "ethan": 2302 + }, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 2048, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_talker_text", + "moe_intermediate_size": 384, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 6, + "num_hidden_layers": 20, + "num_key_value_heads": 2, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 768, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 3072 + }, + "thinker_hidden_size": 2048, + "video_token_id": 151656, + "vision_start_token_id": 151652 + }, + "thinker_config": { + "audio_config": { + "_name_or_path": "", + "activation_dropout": 0, + "activation_function": "gelu", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "conv_chunksize": 500, + "cross_attention_hidden_size": null, + "d_model": 1280, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "downsample_hidden_size": 480, + "dropout": 0, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layers": 32, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_source_positions": 1500, + "min_length": 0, + "model_type": "qwen3_omni_moe_audio_encoder", + "n_window": 50, + "n_window_infer": 800, + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 32, + "num_mel_bins": 128, + "num_return_sequences": 1, + "output_attentions": false, + "output_dim": 2048, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "scale_embedding": false, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "dtype": "bfloat16", + "image_token_id": 151655, + "initializer_range": 0.02, + "model_type": "qwen3_omni_moe_thinker", + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0.0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 768, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_text", + "moe_intermediate_size": 768, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 32, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 48, + "num_key_value_heads": 4, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 0, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_qk_norm": true, + "use_sliding_window": false, + "vocab_size": 152064 + }, + "user_token_id": 872, + "video_token_id": 151656, + "vision_config": { + "_name_or_path": "", + "add_cross_attention": false, + "apply_vit_abs_pos_embed": true, + "architectures": null, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "deepstack_visual_indexes": [ + 8, + 16, + 24 + ], + "depth": 27, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 768, + "in_channels": 3, + "in_chans": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "model_type": "qwen3_omni_moe_vision_encoder", + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_heads": 16, + "num_position_embeddings": 2304, + "num_return_sequences": 1, + "out_hidden_size": 2048, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 16, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "spatial_merge_size": 2, + "spatial_patch_size": 16, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "temporal_patch_size": 2, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "tokens_per_second": 2, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 + }, + "transformers_version": "4.57.3", + "tts_bos_token_id": 151672, + "tts_eos_token_id": 151673, + "tts_pad_token_id": 151671, + "user_token_id": 872 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/iter_0022000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/iter_0022000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..1acc9b8f86016f78501a48ab7ce8f1b748790b38 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/iter_0022000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:556b06ad9cb54bfa3582dcd694cb3df151eb38385d745b5b2114ad3ff0d3e496 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..4d597fb33ecbc4b6403119ac62baac6d60e35db3 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +22000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7df5b45379044a680218cb2b7ed4d01a6633f537 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/preprocessor_config.json @@ -0,0 +1,30 @@ +{ + "dither": 0.0, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_pixels": 12845056, + "merge_size": 2, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "return_attention_mask": true, + "sampling_rate": 16000, + "temporal_patch_size": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/special_tokens_map.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..fca9909b535ff91b68474e5ee7258e1574e60d1e --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/special_tokens_map.json @@ -0,0 +1,44 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "image_token": "<|image_pad|>", + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/tokenizer_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1491f11cc7fc66a763bcda6f4f6b8c3d026845 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/tokenizer_config.json @@ -0,0 +1,317 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151669": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151670": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151675": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "image_token": "<|image_pad|>", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3OmniMoeProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/video_preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6e0fb77bf83924807f06b6f314210795f76c2e42 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000-merged/video_preprocessor_config.json @@ -0,0 +1,54 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "dither": 0.0, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": false, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "fps": null, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "max_pixels": 12845056, + "merge_size": 2, + "min_frames": 4, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "num_frames": null, + "pad_size": null, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_attention_mask": true, + "return_metadata": false, + "sampling_rate": 16000, + "size": { + "longest_edge": 12845056, + "shortest_edge": 3136 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen2VLVideoProcessor" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000/iter_0022000/.metadata b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000/iter_0022000/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..8b45b4585cdba100b4367d6a214c10ff9b98acae Binary files /dev/null and b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000/iter_0022000/.metadata differ diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000/iter_0022000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000/iter_0022000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..1acc9b8f86016f78501a48ab7ce8f1b748790b38 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000/iter_0022000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:556b06ad9cb54bfa3582dcd694cb3df151eb38385d745b5b2114ad3ff0d3e496 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000/iter_0022000/metadata.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000/iter_0022000/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000/iter_0022000/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..4d597fb33ecbc4b6403119ac62baac6d60e35db3 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22000/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +22000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/added_tokens.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..c962f0ea74c2dc3172f28f532b632288bf7d56c3 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/added_tokens.json @@ -0,0 +1,35 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "": 151671, + "": 151672, + "": 151674, + "": 151673, + "<|audio_end|>": 151670, + "<|audio_pad|>": 151675, + "<|audio_start|>": 151669, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/chat_template.jinja b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1efcce1402ae5d1d61d4a4cf56accea5921251cf --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/chat_template.jinja @@ -0,0 +1,122 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- "<|vision_start|><|image_pad|><|vision_end|>" }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- "<|audio_start|><|audio_pad|><|audio_end|>" }} + {%- elif content.type == 'video' or 'video' in content %} + {{- "<|vision_start|><|video_pad|><|vision_end|>" }} + {%- elif content.type == 'text' %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} + {{- '\n\n' }} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|image_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- '<|im_start|>system\n' +"<|audio_start|><|audio_pad|><|audio_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'video' or 'video' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|video_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'text' %} + {{- '<|im_start|>system\n' +content.text+ '<|im_end|>\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = namespace(text="") %} + {%- for mcontent in message.content %} + {%- if mcontent.type == 'image' or 'image' in mcontent or 'image_url' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|image_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'audio' or 'audio' in mcontent or 'audio_url' in mcontent %} + {%- set content.text = content.text~"<|audio_start|><|audio_pad|><|audio_end|>" %} + {%- elif mcontent.type == 'video' or 'video' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|video_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'text' %} + {%- set content.text = content.text~mcontent.text %} + {%- endif %} + {%- endfor %} + {%- set content = content.text %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = "" %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip("\n") + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %}{{- '\n' }}{%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}{{- '<|im_start|>user' }}{%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}{{- '<|im_end|>\n' }}{%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %}{{- '\n\n\n\n' }}{%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6dce3d213c49a2ea04a5bc9ba76b53e440cab76c --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/config.json @@ -0,0 +1,552 @@ +{ + "architectures": [ + "Qwen3OmniMoeForConditionalGeneration" + ], + "assistant_token_id": 77091, + "code2wav_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "codebook_dim": 512, + "codebook_size": 2048, + "decoder_dim": 1536, + "hidden_act": "silu", + "hidden_size": 1024, + "intermediate_size": 3072, + "layer_scale_initial_scale": 0.01, + "max_position_embeddings": 8000, + "model_type": "", + "num_attention_heads": 16, + "num_hidden_layers": 8, + "num_key_value_heads": 16, + "num_quantizers": 16, + "num_semantic_quantizers": 1, + "rms_norm_eps": 1e-05, + "rope_theta": 10000, + "semantic_codebook_size": 4096, + "sliding_window": 72, + "upsample_rates": [ + 8, + 5, + 4, + 3 + ], + "upsampling_ratios": [ + 2, + 2 + ], + "vector_quantization_hidden_dimension": 512 + }, + "dtype": "bfloat16", + "enable_audio_output": true, + "hidden_size": 1024, + "im_end_token_id": 151645, + "im_start_token_id": 151644, + "keys_to_ignore_at_inference": [ + "past_key_values" + ], + "model_type": "qwen3_omni_moe", + "system_token_id": 8948, + "talker_config": { + "accept_hidden_layer": 24, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "code_predictor_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": null, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 3072, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 32768, + "max_window_layers": 28, + "min_length": 0, + "model_type": "qwen3_omni_moe_talker_code_predictor", + "no_repeat_ngram_size": 0, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_code_groups": 16, + "num_hidden_layers": 5, + "num_key_value_heads": 8, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sep_token_id": null, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 2048 + }, + "codec_bos_id": 2149, + "codec_eos_token_id": 2150, + "codec_nothink_id": 2155, + "codec_pad_id": 2148, + "codec_think_bos_id": 2156, + "codec_think_eos_id": 2157, + "image_token_id": 151655, + "model_type": "", + "num_code_groups": 16, + "output_router_logits": false, + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "spatial_merge_size": 2, + "speaker_id": { + "aiden": 2303, + "chelsie": 2301, + "ethan": 2302 + }, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 2048, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_talker_text", + "moe_intermediate_size": 384, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 6, + "num_hidden_layers": 20, + "num_key_value_heads": 2, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 768, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 3072 + }, + "thinker_hidden_size": 2048, + "video_token_id": 151656, + "vision_start_token_id": 151652 + }, + "thinker_config": { + "audio_config": { + "_name_or_path": "", + "activation_dropout": 0, + "activation_function": "gelu", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "conv_chunksize": 500, + "cross_attention_hidden_size": null, + "d_model": 1280, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "downsample_hidden_size": 480, + "dropout": 0, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layers": 32, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_source_positions": 1500, + "min_length": 0, + "model_type": "qwen3_omni_moe_audio_encoder", + "n_window": 50, + "n_window_infer": 800, + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 32, + "num_mel_bins": 128, + "num_return_sequences": 1, + "output_attentions": false, + "output_dim": 2048, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "scale_embedding": false, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "dtype": "bfloat16", + "image_token_id": 151655, + "initializer_range": 0.02, + "model_type": "qwen3_omni_moe_thinker", + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0.0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 768, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_text", + "moe_intermediate_size": 768, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 32, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 48, + "num_key_value_heads": 4, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 0, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_qk_norm": true, + "use_sliding_window": false, + "vocab_size": 152064 + }, + "user_token_id": 872, + "video_token_id": 151656, + "vision_config": { + "_name_or_path": "", + "add_cross_attention": false, + "apply_vit_abs_pos_embed": true, + "architectures": null, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "deepstack_visual_indexes": [ + 8, + 16, + 24 + ], + "depth": 27, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 768, + "in_channels": 3, + "in_chans": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "model_type": "qwen3_omni_moe_vision_encoder", + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_heads": 16, + "num_position_embeddings": 2304, + "num_return_sequences": 1, + "out_hidden_size": 2048, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 16, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "spatial_merge_size": 2, + "spatial_patch_size": 16, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "temporal_patch_size": 2, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "tokens_per_second": 2, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 + }, + "transformers_version": "4.57.3", + "tts_bos_token_id": 151672, + "tts_eos_token_id": 151673, + "tts_pad_token_id": 151671, + "user_token_id": 872 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/iter_0022905/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/iter_0022905/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e83c55c3186a2d1006f50d5f5f5df6e66f34021 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/iter_0022905/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:155f827cb26aa55418033adb0ec2d2c6e6ffa103372e32634de0460e5d74d263 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..ffcd579c91c6a6247c31e521a5b6a2561527ce08 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +22905 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7df5b45379044a680218cb2b7ed4d01a6633f537 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/preprocessor_config.json @@ -0,0 +1,30 @@ +{ + "dither": 0.0, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_pixels": 12845056, + "merge_size": 2, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "return_attention_mask": true, + "sampling_rate": 16000, + "temporal_patch_size": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/special_tokens_map.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..fca9909b535ff91b68474e5ee7258e1574e60d1e --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/special_tokens_map.json @@ -0,0 +1,44 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "image_token": "<|image_pad|>", + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/tokenizer_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1491f11cc7fc66a763bcda6f4f6b8c3d026845 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/tokenizer_config.json @@ -0,0 +1,317 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151669": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151670": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151675": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "image_token": "<|image_pad|>", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3OmniMoeProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/video_preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6e0fb77bf83924807f06b6f314210795f76c2e42 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905-merged/video_preprocessor_config.json @@ -0,0 +1,54 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "dither": 0.0, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": false, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "fps": null, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "max_pixels": 12845056, + "merge_size": 2, + "min_frames": 4, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "num_frames": null, + "pad_size": null, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_attention_mask": true, + "return_metadata": false, + "sampling_rate": 16000, + "size": { + "longest_edge": 12845056, + "shortest_edge": 3136 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen2VLVideoProcessor" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905/iter_0022905/.metadata b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905/iter_0022905/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..bb61dc8226aafb6c50b7bc7edb708718b2eb28a2 Binary files /dev/null and b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905/iter_0022905/.metadata differ diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905/iter_0022905/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905/iter_0022905/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e83c55c3186a2d1006f50d5f5f5df6e66f34021 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905/iter_0022905/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:155f827cb26aa55418033adb0ec2d2c6e6ffa103372e32634de0460e5d74d263 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905/iter_0022905/metadata.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905/iter_0022905/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905/iter_0022905/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..ffcd579c91c6a6247c31e521a5b6a2561527ce08 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-22905/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +22905 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/added_tokens.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..c962f0ea74c2dc3172f28f532b632288bf7d56c3 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/added_tokens.json @@ -0,0 +1,35 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "": 151671, + "": 151672, + "": 151674, + "": 151673, + "<|audio_end|>": 151670, + "<|audio_pad|>": 151675, + "<|audio_start|>": 151669, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/chat_template.jinja b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1efcce1402ae5d1d61d4a4cf56accea5921251cf --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/chat_template.jinja @@ -0,0 +1,122 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- "<|vision_start|><|image_pad|><|vision_end|>" }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- "<|audio_start|><|audio_pad|><|audio_end|>" }} + {%- elif content.type == 'video' or 'video' in content %} + {{- "<|vision_start|><|video_pad|><|vision_end|>" }} + {%- elif content.type == 'text' %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} + {{- '\n\n' }} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|image_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- '<|im_start|>system\n' +"<|audio_start|><|audio_pad|><|audio_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'video' or 'video' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|video_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'text' %} + {{- '<|im_start|>system\n' +content.text+ '<|im_end|>\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = namespace(text="") %} + {%- for mcontent in message.content %} + {%- if mcontent.type == 'image' or 'image' in mcontent or 'image_url' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|image_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'audio' or 'audio' in mcontent or 'audio_url' in mcontent %} + {%- set content.text = content.text~"<|audio_start|><|audio_pad|><|audio_end|>" %} + {%- elif mcontent.type == 'video' or 'video' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|video_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'text' %} + {%- set content.text = content.text~mcontent.text %} + {%- endif %} + {%- endfor %} + {%- set content = content.text %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = "" %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip("\n") + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %}{{- '\n' }}{%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}{{- '<|im_start|>user' }}{%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}{{- '<|im_end|>\n' }}{%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %}{{- '\n\n\n\n' }}{%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6dce3d213c49a2ea04a5bc9ba76b53e440cab76c --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/config.json @@ -0,0 +1,552 @@ +{ + "architectures": [ + "Qwen3OmniMoeForConditionalGeneration" + ], + "assistant_token_id": 77091, + "code2wav_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "codebook_dim": 512, + "codebook_size": 2048, + "decoder_dim": 1536, + "hidden_act": "silu", + "hidden_size": 1024, + "intermediate_size": 3072, + "layer_scale_initial_scale": 0.01, + "max_position_embeddings": 8000, + "model_type": "", + "num_attention_heads": 16, + "num_hidden_layers": 8, + "num_key_value_heads": 16, + "num_quantizers": 16, + "num_semantic_quantizers": 1, + "rms_norm_eps": 1e-05, + "rope_theta": 10000, + "semantic_codebook_size": 4096, + "sliding_window": 72, + "upsample_rates": [ + 8, + 5, + 4, + 3 + ], + "upsampling_ratios": [ + 2, + 2 + ], + "vector_quantization_hidden_dimension": 512 + }, + "dtype": "bfloat16", + "enable_audio_output": true, + "hidden_size": 1024, + "im_end_token_id": 151645, + "im_start_token_id": 151644, + "keys_to_ignore_at_inference": [ + "past_key_values" + ], + "model_type": "qwen3_omni_moe", + "system_token_id": 8948, + "talker_config": { + "accept_hidden_layer": 24, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "code_predictor_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": null, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 3072, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 32768, + "max_window_layers": 28, + "min_length": 0, + "model_type": "qwen3_omni_moe_talker_code_predictor", + "no_repeat_ngram_size": 0, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_code_groups": 16, + "num_hidden_layers": 5, + "num_key_value_heads": 8, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sep_token_id": null, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 2048 + }, + "codec_bos_id": 2149, + "codec_eos_token_id": 2150, + "codec_nothink_id": 2155, + "codec_pad_id": 2148, + "codec_think_bos_id": 2156, + "codec_think_eos_id": 2157, + "image_token_id": 151655, + "model_type": "", + "num_code_groups": 16, + "output_router_logits": false, + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "spatial_merge_size": 2, + "speaker_id": { + "aiden": 2303, + "chelsie": 2301, + "ethan": 2302 + }, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 2048, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_talker_text", + "moe_intermediate_size": 384, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 6, + "num_hidden_layers": 20, + "num_key_value_heads": 2, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 768, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 3072 + }, + "thinker_hidden_size": 2048, + "video_token_id": 151656, + "vision_start_token_id": 151652 + }, + "thinker_config": { + "audio_config": { + "_name_or_path": "", + "activation_dropout": 0, + "activation_function": "gelu", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "conv_chunksize": 500, + "cross_attention_hidden_size": null, + "d_model": 1280, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "downsample_hidden_size": 480, + "dropout": 0, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layers": 32, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_source_positions": 1500, + "min_length": 0, + "model_type": "qwen3_omni_moe_audio_encoder", + "n_window": 50, + "n_window_infer": 800, + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 32, + "num_mel_bins": 128, + "num_return_sequences": 1, + "output_attentions": false, + "output_dim": 2048, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "scale_embedding": false, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "dtype": "bfloat16", + "image_token_id": 151655, + "initializer_range": 0.02, + "model_type": "qwen3_omni_moe_thinker", + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0.0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 768, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_text", + "moe_intermediate_size": 768, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 32, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 48, + "num_key_value_heads": 4, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 0, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_qk_norm": true, + "use_sliding_window": false, + "vocab_size": 152064 + }, + "user_token_id": 872, + "video_token_id": 151656, + "vision_config": { + "_name_or_path": "", + "add_cross_attention": false, + "apply_vit_abs_pos_embed": true, + "architectures": null, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "deepstack_visual_indexes": [ + 8, + 16, + 24 + ], + "depth": 27, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 768, + "in_channels": 3, + "in_chans": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "model_type": "qwen3_omni_moe_vision_encoder", + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_heads": 16, + "num_position_embeddings": 2304, + "num_return_sequences": 1, + "out_hidden_size": 2048, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 16, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "spatial_merge_size": 2, + "spatial_patch_size": 16, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "temporal_patch_size": 2, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "tokens_per_second": 2, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 + }, + "transformers_version": "4.57.3", + "tts_bos_token_id": 151672, + "tts_eos_token_id": 151673, + "tts_pad_token_id": 151671, + "user_token_id": 872 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/iter_0003000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/iter_0003000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..21ce56d42d501cf046f421b92870f9d5d5da7ede --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/iter_0003000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd48654a995c18393d32f809d5a4f3d6c9a1933b361fb3d2b604fe4aa0c879be +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..8da85a226acb01bedfea3928242822fcdc231429 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +3000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7df5b45379044a680218cb2b7ed4d01a6633f537 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/preprocessor_config.json @@ -0,0 +1,30 @@ +{ + "dither": 0.0, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_pixels": 12845056, + "merge_size": 2, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "return_attention_mask": true, + "sampling_rate": 16000, + "temporal_patch_size": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/special_tokens_map.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..fca9909b535ff91b68474e5ee7258e1574e60d1e --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/special_tokens_map.json @@ -0,0 +1,44 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "image_token": "<|image_pad|>", + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/tokenizer_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1491f11cc7fc66a763bcda6f4f6b8c3d026845 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/tokenizer_config.json @@ -0,0 +1,317 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151669": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151670": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151675": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "image_token": "<|image_pad|>", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3OmniMoeProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/video_preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6e0fb77bf83924807f06b6f314210795f76c2e42 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000-merged/video_preprocessor_config.json @@ -0,0 +1,54 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "dither": 0.0, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": false, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "fps": null, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "max_pixels": 12845056, + "merge_size": 2, + "min_frames": 4, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "num_frames": null, + "pad_size": null, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_attention_mask": true, + "return_metadata": false, + "sampling_rate": 16000, + "size": { + "longest_edge": 12845056, + "shortest_edge": 3136 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen2VLVideoProcessor" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000/iter_0003000/.metadata b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000/iter_0003000/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..8df6261198898b437e8878b009b201422bbd98b1 Binary files /dev/null and b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000/iter_0003000/.metadata differ diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000/iter_0003000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000/iter_0003000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..21ce56d42d501cf046f421b92870f9d5d5da7ede --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000/iter_0003000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd48654a995c18393d32f809d5a4f3d6c9a1933b361fb3d2b604fe4aa0c879be +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000/iter_0003000/metadata.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000/iter_0003000/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000/iter_0003000/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..8da85a226acb01bedfea3928242822fcdc231429 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-3000/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +3000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/added_tokens.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..c962f0ea74c2dc3172f28f532b632288bf7d56c3 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/added_tokens.json @@ -0,0 +1,35 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "": 151671, + "": 151672, + "": 151674, + "": 151673, + "<|audio_end|>": 151670, + "<|audio_pad|>": 151675, + "<|audio_start|>": 151669, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/chat_template.jinja b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1efcce1402ae5d1d61d4a4cf56accea5921251cf --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/chat_template.jinja @@ -0,0 +1,122 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- "<|vision_start|><|image_pad|><|vision_end|>" }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- "<|audio_start|><|audio_pad|><|audio_end|>" }} + {%- elif content.type == 'video' or 'video' in content %} + {{- "<|vision_start|><|video_pad|><|vision_end|>" }} + {%- elif content.type == 'text' %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} + {{- '\n\n' }} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|image_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- '<|im_start|>system\n' +"<|audio_start|><|audio_pad|><|audio_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'video' or 'video' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|video_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'text' %} + {{- '<|im_start|>system\n' +content.text+ '<|im_end|>\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = namespace(text="") %} + {%- for mcontent in message.content %} + {%- if mcontent.type == 'image' or 'image' in mcontent or 'image_url' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|image_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'audio' or 'audio' in mcontent or 'audio_url' in mcontent %} + {%- set content.text = content.text~"<|audio_start|><|audio_pad|><|audio_end|>" %} + {%- elif mcontent.type == 'video' or 'video' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|video_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'text' %} + {%- set content.text = content.text~mcontent.text %} + {%- endif %} + {%- endfor %} + {%- set content = content.text %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = "" %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip("\n") + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %}{{- '\n' }}{%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}{{- '<|im_start|>user' }}{%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}{{- '<|im_end|>\n' }}{%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %}{{- '\n\n\n\n' }}{%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6dce3d213c49a2ea04a5bc9ba76b53e440cab76c --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/config.json @@ -0,0 +1,552 @@ +{ + "architectures": [ + "Qwen3OmniMoeForConditionalGeneration" + ], + "assistant_token_id": 77091, + "code2wav_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "codebook_dim": 512, + "codebook_size": 2048, + "decoder_dim": 1536, + "hidden_act": "silu", + "hidden_size": 1024, + "intermediate_size": 3072, + "layer_scale_initial_scale": 0.01, + "max_position_embeddings": 8000, + "model_type": "", + "num_attention_heads": 16, + "num_hidden_layers": 8, + "num_key_value_heads": 16, + "num_quantizers": 16, + "num_semantic_quantizers": 1, + "rms_norm_eps": 1e-05, + "rope_theta": 10000, + "semantic_codebook_size": 4096, + "sliding_window": 72, + "upsample_rates": [ + 8, + 5, + 4, + 3 + ], + "upsampling_ratios": [ + 2, + 2 + ], + "vector_quantization_hidden_dimension": 512 + }, + "dtype": "bfloat16", + "enable_audio_output": true, + "hidden_size": 1024, + "im_end_token_id": 151645, + "im_start_token_id": 151644, + "keys_to_ignore_at_inference": [ + "past_key_values" + ], + "model_type": "qwen3_omni_moe", + "system_token_id": 8948, + "talker_config": { + "accept_hidden_layer": 24, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "code_predictor_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": null, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 3072, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 32768, + "max_window_layers": 28, + "min_length": 0, + "model_type": "qwen3_omni_moe_talker_code_predictor", + "no_repeat_ngram_size": 0, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_code_groups": 16, + "num_hidden_layers": 5, + "num_key_value_heads": 8, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sep_token_id": null, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 2048 + }, + "codec_bos_id": 2149, + "codec_eos_token_id": 2150, + "codec_nothink_id": 2155, + "codec_pad_id": 2148, + "codec_think_bos_id": 2156, + "codec_think_eos_id": 2157, + "image_token_id": 151655, + "model_type": "", + "num_code_groups": 16, + "output_router_logits": false, + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "spatial_merge_size": 2, + "speaker_id": { + "aiden": 2303, + "chelsie": 2301, + "ethan": 2302 + }, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 2048, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_talker_text", + "moe_intermediate_size": 384, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 6, + "num_hidden_layers": 20, + "num_key_value_heads": 2, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 768, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 3072 + }, + "thinker_hidden_size": 2048, + "video_token_id": 151656, + "vision_start_token_id": 151652 + }, + "thinker_config": { + "audio_config": { + "_name_or_path": "", + "activation_dropout": 0, + "activation_function": "gelu", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "conv_chunksize": 500, + "cross_attention_hidden_size": null, + "d_model": 1280, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "downsample_hidden_size": 480, + "dropout": 0, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layers": 32, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_source_positions": 1500, + "min_length": 0, + "model_type": "qwen3_omni_moe_audio_encoder", + "n_window": 50, + "n_window_infer": 800, + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 32, + "num_mel_bins": 128, + "num_return_sequences": 1, + "output_attentions": false, + "output_dim": 2048, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "scale_embedding": false, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "dtype": "bfloat16", + "image_token_id": 151655, + "initializer_range": 0.02, + "model_type": "qwen3_omni_moe_thinker", + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0.0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 768, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_text", + "moe_intermediate_size": 768, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 32, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 48, + "num_key_value_heads": 4, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 0, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_qk_norm": true, + "use_sliding_window": false, + "vocab_size": 152064 + }, + "user_token_id": 872, + "video_token_id": 151656, + "vision_config": { + "_name_or_path": "", + "add_cross_attention": false, + "apply_vit_abs_pos_embed": true, + "architectures": null, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "deepstack_visual_indexes": [ + 8, + 16, + 24 + ], + "depth": 27, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 768, + "in_channels": 3, + "in_chans": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "model_type": "qwen3_omni_moe_vision_encoder", + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_heads": 16, + "num_position_embeddings": 2304, + "num_return_sequences": 1, + "out_hidden_size": 2048, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 16, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "spatial_merge_size": 2, + "spatial_patch_size": 16, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "temporal_patch_size": 2, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "tokens_per_second": 2, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 + }, + "transformers_version": "4.57.3", + "tts_bos_token_id": 151672, + "tts_eos_token_id": 151673, + "tts_pad_token_id": 151671, + "user_token_id": 872 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/iter_0004000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/iter_0004000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..7fa5a22e349ded85e33a4d808e999ee27c73a2f0 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/iter_0004000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3625e9f79b297929a412ecdd4150ec9d663aab800c28ceab7b4e5c6c7f5703be +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..a2113715a3e842865ae400f6d08eb9a7b4898256 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +4000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7df5b45379044a680218cb2b7ed4d01a6633f537 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/preprocessor_config.json @@ -0,0 +1,30 @@ +{ + "dither": 0.0, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_pixels": 12845056, + "merge_size": 2, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "return_attention_mask": true, + "sampling_rate": 16000, + "temporal_patch_size": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/special_tokens_map.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..fca9909b535ff91b68474e5ee7258e1574e60d1e --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/special_tokens_map.json @@ -0,0 +1,44 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "image_token": "<|image_pad|>", + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/tokenizer_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1491f11cc7fc66a763bcda6f4f6b8c3d026845 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/tokenizer_config.json @@ -0,0 +1,317 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151669": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151670": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151675": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "image_token": "<|image_pad|>", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3OmniMoeProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/video_preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6e0fb77bf83924807f06b6f314210795f76c2e42 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000-merged/video_preprocessor_config.json @@ -0,0 +1,54 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "dither": 0.0, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": false, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "fps": null, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "max_pixels": 12845056, + "merge_size": 2, + "min_frames": 4, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "num_frames": null, + "pad_size": null, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_attention_mask": true, + "return_metadata": false, + "sampling_rate": 16000, + "size": { + "longest_edge": 12845056, + "shortest_edge": 3136 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen2VLVideoProcessor" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000/iter_0004000/.metadata b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000/iter_0004000/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..ad822fe6f3b0f299d643671048c184e9e49c38c8 Binary files /dev/null and b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000/iter_0004000/.metadata differ diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000/iter_0004000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000/iter_0004000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..7fa5a22e349ded85e33a4d808e999ee27c73a2f0 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000/iter_0004000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3625e9f79b297929a412ecdd4150ec9d663aab800c28ceab7b4e5c6c7f5703be +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000/iter_0004000/metadata.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000/iter_0004000/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000/iter_0004000/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..a2113715a3e842865ae400f6d08eb9a7b4898256 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-4000/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +4000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/added_tokens.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..c962f0ea74c2dc3172f28f532b632288bf7d56c3 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/added_tokens.json @@ -0,0 +1,35 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "": 151671, + "": 151672, + "": 151674, + "": 151673, + "<|audio_end|>": 151670, + "<|audio_pad|>": 151675, + "<|audio_start|>": 151669, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/chat_template.jinja b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1efcce1402ae5d1d61d4a4cf56accea5921251cf --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/chat_template.jinja @@ -0,0 +1,122 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- "<|vision_start|><|image_pad|><|vision_end|>" }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- "<|audio_start|><|audio_pad|><|audio_end|>" }} + {%- elif content.type == 'video' or 'video' in content %} + {{- "<|vision_start|><|video_pad|><|vision_end|>" }} + {%- elif content.type == 'text' %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} + {{- '\n\n' }} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|image_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- '<|im_start|>system\n' +"<|audio_start|><|audio_pad|><|audio_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'video' or 'video' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|video_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'text' %} + {{- '<|im_start|>system\n' +content.text+ '<|im_end|>\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = namespace(text="") %} + {%- for mcontent in message.content %} + {%- if mcontent.type == 'image' or 'image' in mcontent or 'image_url' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|image_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'audio' or 'audio' in mcontent or 'audio_url' in mcontent %} + {%- set content.text = content.text~"<|audio_start|><|audio_pad|><|audio_end|>" %} + {%- elif mcontent.type == 'video' or 'video' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|video_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'text' %} + {%- set content.text = content.text~mcontent.text %} + {%- endif %} + {%- endfor %} + {%- set content = content.text %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = "" %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip("\n") + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %}{{- '\n' }}{%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}{{- '<|im_start|>user' }}{%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}{{- '<|im_end|>\n' }}{%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %}{{- '\n\n\n\n' }}{%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6dce3d213c49a2ea04a5bc9ba76b53e440cab76c --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/config.json @@ -0,0 +1,552 @@ +{ + "architectures": [ + "Qwen3OmniMoeForConditionalGeneration" + ], + "assistant_token_id": 77091, + "code2wav_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "codebook_dim": 512, + "codebook_size": 2048, + "decoder_dim": 1536, + "hidden_act": "silu", + "hidden_size": 1024, + "intermediate_size": 3072, + "layer_scale_initial_scale": 0.01, + "max_position_embeddings": 8000, + "model_type": "", + "num_attention_heads": 16, + "num_hidden_layers": 8, + "num_key_value_heads": 16, + "num_quantizers": 16, + "num_semantic_quantizers": 1, + "rms_norm_eps": 1e-05, + "rope_theta": 10000, + "semantic_codebook_size": 4096, + "sliding_window": 72, + "upsample_rates": [ + 8, + 5, + 4, + 3 + ], + "upsampling_ratios": [ + 2, + 2 + ], + "vector_quantization_hidden_dimension": 512 + }, + "dtype": "bfloat16", + "enable_audio_output": true, + "hidden_size": 1024, + "im_end_token_id": 151645, + "im_start_token_id": 151644, + "keys_to_ignore_at_inference": [ + "past_key_values" + ], + "model_type": "qwen3_omni_moe", + "system_token_id": 8948, + "talker_config": { + "accept_hidden_layer": 24, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "code_predictor_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": null, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 3072, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 32768, + "max_window_layers": 28, + "min_length": 0, + "model_type": "qwen3_omni_moe_talker_code_predictor", + "no_repeat_ngram_size": 0, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_code_groups": 16, + "num_hidden_layers": 5, + "num_key_value_heads": 8, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sep_token_id": null, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 2048 + }, + "codec_bos_id": 2149, + "codec_eos_token_id": 2150, + "codec_nothink_id": 2155, + "codec_pad_id": 2148, + "codec_think_bos_id": 2156, + "codec_think_eos_id": 2157, + "image_token_id": 151655, + "model_type": "", + "num_code_groups": 16, + "output_router_logits": false, + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "spatial_merge_size": 2, + "speaker_id": { + "aiden": 2303, + "chelsie": 2301, + "ethan": 2302 + }, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 2048, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_talker_text", + "moe_intermediate_size": 384, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 6, + "num_hidden_layers": 20, + "num_key_value_heads": 2, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 768, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 3072 + }, + "thinker_hidden_size": 2048, + "video_token_id": 151656, + "vision_start_token_id": 151652 + }, + "thinker_config": { + "audio_config": { + "_name_or_path": "", + "activation_dropout": 0, + "activation_function": "gelu", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "conv_chunksize": 500, + "cross_attention_hidden_size": null, + "d_model": 1280, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "downsample_hidden_size": 480, + "dropout": 0, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layers": 32, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_source_positions": 1500, + "min_length": 0, + "model_type": "qwen3_omni_moe_audio_encoder", + "n_window": 50, + "n_window_infer": 800, + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 32, + "num_mel_bins": 128, + "num_return_sequences": 1, + "output_attentions": false, + "output_dim": 2048, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "scale_embedding": false, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "dtype": "bfloat16", + "image_token_id": 151655, + "initializer_range": 0.02, + "model_type": "qwen3_omni_moe_thinker", + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0.0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 768, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_text", + "moe_intermediate_size": 768, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 32, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 48, + "num_key_value_heads": 4, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 0, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_qk_norm": true, + "use_sliding_window": false, + "vocab_size": 152064 + }, + "user_token_id": 872, + "video_token_id": 151656, + "vision_config": { + "_name_or_path": "", + "add_cross_attention": false, + "apply_vit_abs_pos_embed": true, + "architectures": null, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "deepstack_visual_indexes": [ + 8, + 16, + 24 + ], + "depth": 27, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 768, + "in_channels": 3, + "in_chans": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "model_type": "qwen3_omni_moe_vision_encoder", + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_heads": 16, + "num_position_embeddings": 2304, + "num_return_sequences": 1, + "out_hidden_size": 2048, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 16, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "spatial_merge_size": 2, + "spatial_patch_size": 16, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "temporal_patch_size": 2, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "tokens_per_second": 2, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 + }, + "transformers_version": "4.57.3", + "tts_bos_token_id": 151672, + "tts_eos_token_id": 151673, + "tts_pad_token_id": 151671, + "user_token_id": 872 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/iter_0005000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/iter_0005000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab7b1f2c72173e358c56a7bf12fd2e80ce651722 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/iter_0005000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d154c9f793e8bc7f27c1fcfaf1f5ebb7e172287626d0b5889655a84b047ddd5 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..0b3e0a69a8c7977c396c644a388038e7ebe3fc3e --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +5000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7df5b45379044a680218cb2b7ed4d01a6633f537 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/preprocessor_config.json @@ -0,0 +1,30 @@ +{ + "dither": 0.0, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_pixels": 12845056, + "merge_size": 2, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "return_attention_mask": true, + "sampling_rate": 16000, + "temporal_patch_size": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/special_tokens_map.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..fca9909b535ff91b68474e5ee7258e1574e60d1e --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/special_tokens_map.json @@ -0,0 +1,44 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "image_token": "<|image_pad|>", + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/tokenizer_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1491f11cc7fc66a763bcda6f4f6b8c3d026845 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/tokenizer_config.json @@ -0,0 +1,317 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151669": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151670": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151675": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "image_token": "<|image_pad|>", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3OmniMoeProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/video_preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6e0fb77bf83924807f06b6f314210795f76c2e42 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000-merged/video_preprocessor_config.json @@ -0,0 +1,54 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "dither": 0.0, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": false, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "fps": null, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "max_pixels": 12845056, + "merge_size": 2, + "min_frames": 4, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "num_frames": null, + "pad_size": null, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_attention_mask": true, + "return_metadata": false, + "sampling_rate": 16000, + "size": { + "longest_edge": 12845056, + "shortest_edge": 3136 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen2VLVideoProcessor" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000/iter_0005000/.metadata b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000/iter_0005000/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..25c397da5cdc3fee84395ca18d52fbb58bf270f5 Binary files /dev/null and b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000/iter_0005000/.metadata differ diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000/iter_0005000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000/iter_0005000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab7b1f2c72173e358c56a7bf12fd2e80ce651722 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000/iter_0005000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d154c9f793e8bc7f27c1fcfaf1f5ebb7e172287626d0b5889655a84b047ddd5 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000/iter_0005000/metadata.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000/iter_0005000/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000/iter_0005000/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..0b3e0a69a8c7977c396c644a388038e7ebe3fc3e --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-5000/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +5000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/added_tokens.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..c962f0ea74c2dc3172f28f532b632288bf7d56c3 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/added_tokens.json @@ -0,0 +1,35 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "": 151671, + "": 151672, + "": 151674, + "": 151673, + "<|audio_end|>": 151670, + "<|audio_pad|>": 151675, + "<|audio_start|>": 151669, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/chat_template.jinja b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1efcce1402ae5d1d61d4a4cf56accea5921251cf --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/chat_template.jinja @@ -0,0 +1,122 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- "<|vision_start|><|image_pad|><|vision_end|>" }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- "<|audio_start|><|audio_pad|><|audio_end|>" }} + {%- elif content.type == 'video' or 'video' in content %} + {{- "<|vision_start|><|video_pad|><|vision_end|>" }} + {%- elif content.type == 'text' %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} + {{- '\n\n' }} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|image_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- '<|im_start|>system\n' +"<|audio_start|><|audio_pad|><|audio_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'video' or 'video' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|video_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'text' %} + {{- '<|im_start|>system\n' +content.text+ '<|im_end|>\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = namespace(text="") %} + {%- for mcontent in message.content %} + {%- if mcontent.type == 'image' or 'image' in mcontent or 'image_url' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|image_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'audio' or 'audio' in mcontent or 'audio_url' in mcontent %} + {%- set content.text = content.text~"<|audio_start|><|audio_pad|><|audio_end|>" %} + {%- elif mcontent.type == 'video' or 'video' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|video_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'text' %} + {%- set content.text = content.text~mcontent.text %} + {%- endif %} + {%- endfor %} + {%- set content = content.text %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = "" %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip("\n") + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %}{{- '\n' }}{%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}{{- '<|im_start|>user' }}{%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}{{- '<|im_end|>\n' }}{%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %}{{- '\n\n\n\n' }}{%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6dce3d213c49a2ea04a5bc9ba76b53e440cab76c --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/config.json @@ -0,0 +1,552 @@ +{ + "architectures": [ + "Qwen3OmniMoeForConditionalGeneration" + ], + "assistant_token_id": 77091, + "code2wav_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "codebook_dim": 512, + "codebook_size": 2048, + "decoder_dim": 1536, + "hidden_act": "silu", + "hidden_size": 1024, + "intermediate_size": 3072, + "layer_scale_initial_scale": 0.01, + "max_position_embeddings": 8000, + "model_type": "", + "num_attention_heads": 16, + "num_hidden_layers": 8, + "num_key_value_heads": 16, + "num_quantizers": 16, + "num_semantic_quantizers": 1, + "rms_norm_eps": 1e-05, + "rope_theta": 10000, + "semantic_codebook_size": 4096, + "sliding_window": 72, + "upsample_rates": [ + 8, + 5, + 4, + 3 + ], + "upsampling_ratios": [ + 2, + 2 + ], + "vector_quantization_hidden_dimension": 512 + }, + "dtype": "bfloat16", + "enable_audio_output": true, + "hidden_size": 1024, + "im_end_token_id": 151645, + "im_start_token_id": 151644, + "keys_to_ignore_at_inference": [ + "past_key_values" + ], + "model_type": "qwen3_omni_moe", + "system_token_id": 8948, + "talker_config": { + "accept_hidden_layer": 24, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "code_predictor_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": null, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 3072, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 32768, + "max_window_layers": 28, + "min_length": 0, + "model_type": "qwen3_omni_moe_talker_code_predictor", + "no_repeat_ngram_size": 0, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_code_groups": 16, + "num_hidden_layers": 5, + "num_key_value_heads": 8, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sep_token_id": null, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 2048 + }, + "codec_bos_id": 2149, + "codec_eos_token_id": 2150, + "codec_nothink_id": 2155, + "codec_pad_id": 2148, + "codec_think_bos_id": 2156, + "codec_think_eos_id": 2157, + "image_token_id": 151655, + "model_type": "", + "num_code_groups": 16, + "output_router_logits": false, + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "spatial_merge_size": 2, + "speaker_id": { + "aiden": 2303, + "chelsie": 2301, + "ethan": 2302 + }, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 2048, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_talker_text", + "moe_intermediate_size": 384, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 6, + "num_hidden_layers": 20, + "num_key_value_heads": 2, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 768, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 3072 + }, + "thinker_hidden_size": 2048, + "video_token_id": 151656, + "vision_start_token_id": 151652 + }, + "thinker_config": { + "audio_config": { + "_name_or_path": "", + "activation_dropout": 0, + "activation_function": "gelu", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "conv_chunksize": 500, + "cross_attention_hidden_size": null, + "d_model": 1280, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "downsample_hidden_size": 480, + "dropout": 0, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layers": 32, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_source_positions": 1500, + "min_length": 0, + "model_type": "qwen3_omni_moe_audio_encoder", + "n_window": 50, + "n_window_infer": 800, + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 32, + "num_mel_bins": 128, + "num_return_sequences": 1, + "output_attentions": false, + "output_dim": 2048, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "scale_embedding": false, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "audio_end_token_id": 151670, + "audio_start_token_id": 151669, + "audio_token_id": 151675, + "dtype": "bfloat16", + "image_token_id": 151655, + "initializer_range": 0.02, + "model_type": "qwen3_omni_moe_thinker", + "position_id_per_seconds": 13, + "seconds_per_chunk": 2, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_bias": false, + "attention_dropout": 0.0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_sparse_step": 1, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 768, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 65536, + "min_length": 0, + "mlp_only_layers": [], + "model_type": "qwen3_omni_moe_text", + "moe_intermediate_size": 768, + "no_repeat_ngram_size": 0, + "norm_topk_prob": true, + "num_attention_heads": 32, + "num_beam_groups": 1, + "num_beams": 1, + "num_experts": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 48, + "num_key_value_heads": 4, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_router_logits": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "interleaved": true, + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000, + "router_aux_loss_coef": 0.001, + "sep_token_id": null, + "shared_expert_intermediate_size": 0, + "sliding_window": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_qk_norm": true, + "use_sliding_window": false, + "vocab_size": 152064 + }, + "user_token_id": 872, + "video_token_id": 151656, + "vision_config": { + "_name_or_path": "", + "add_cross_attention": false, + "apply_vit_abs_pos_embed": true, + "architectures": null, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "deepstack_visual_indexes": [ + 8, + 16, + 24 + ], + "depth": 27, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 768, + "in_channels": 3, + "in_chans": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "model_type": "qwen3_omni_moe_vision_encoder", + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_heads": 16, + "num_position_embeddings": 2304, + "num_return_sequences": 1, + "out_hidden_size": 2048, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 16, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "spatial_merge_size": 2, + "spatial_patch_size": 16, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "temporal_patch_size": 2, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "tokens_per_second": 2, + "top_k": 50, + "top_p": 1.0, + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 + }, + "transformers_version": "4.57.3", + "tts_bos_token_id": 151672, + "tts_eos_token_id": 151673, + "tts_pad_token_id": 151671, + "user_token_id": 872 +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/iter_0006000/common.pt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/iter_0006000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..12135f8475ce4c46cce1132ad38306090cf08c5b --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/iter_0006000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a050feab492daf7358de633971f23ac01201a9ee37a31901cf58e82246f25a05 +size 15847 diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..4911b05b53a7dc237714c1f1ed869865bb52ee9b --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +6000 \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7df5b45379044a680218cb2b7ed4d01a6633f537 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/preprocessor_config.json @@ -0,0 +1,30 @@ +{ + "dither": 0.0, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_pixels": 12845056, + "merge_size": 2, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "return_attention_mask": true, + "sampling_rate": 16000, + "temporal_patch_size": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/special_tokens_map.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..fca9909b535ff91b68474e5ee7258e1574e60d1e --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/special_tokens_map.json @@ -0,0 +1,44 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "image_token": "<|image_pad|>", + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/tokenizer_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1491f11cc7fc66a763bcda6f4f6b8c3d026845 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/tokenizer_config.json @@ -0,0 +1,317 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151669": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151670": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151675": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "", + "", + "", + "<|audio_pad|>" + ], + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "image_token": "<|image_pad|>", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3OmniMoeProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/video_preprocessor_config.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6e0fb77bf83924807f06b6f314210795f76c2e42 --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000-merged/video_preprocessor_config.json @@ -0,0 +1,54 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "dither": 0.0, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": false, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "fps": null, + "hop_length": 160, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "max_pixels": 12845056, + "merge_size": 2, + "min_frames": 4, + "min_pixels": 3136, + "n_fft": 400, + "n_samples": 4800000, + "nb_max_frames": 30000, + "num_frames": null, + "pad_size": null, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "processor_class": "Qwen3OmniMoeProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_attention_mask": true, + "return_metadata": false, + "sampling_rate": 16000, + "size": { + "longest_edge": 12845056, + "shortest_edge": 3136 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen2VLVideoProcessor" +} diff --git a/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000/args.json b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000/args.json new file mode 100644 index 0000000000000000000000000000000000000000..a7fe59977ec90a1d35239d5a563d20e22a85e62d --- /dev/null +++ b/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/checkpoint-6000/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "model_type": "qwen3_omni_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_omni", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "default", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": null, + "enable_thinking": null, + "add_non_thinking_prefix": true, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 1, + "global_batch_size": 8, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 1000, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 4, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": true, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 2, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3omni_30b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_qwen3omni", + "wandb_exp_name": "qwen3omni_30b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3-Omni-30B-A3B-Instruct", + "model_info": "ModelInfo(model_type='qwen3_omni_moe', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct', torch_dtype=torch.bfloat16, max_model_len=65536, quant_method=None, quant_bits=None, rope_scaling={'interleaved': True, 'mrope_section': [24, 20, 20], 'rope_type': 'default', 'type': 'default'}, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_omni_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Thinking', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', hf_model_id='Qwen/Qwen3-Omni-30B-A3B-Captioner', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_omni', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_omni', model_arch=MultiModelKeys(arch_name='qwen3_omni', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['thinker.model', 'thinker.lm_head'], aligner=['thinker.audio_tower.proj1', 'thinker.audio_tower.proj2', 'thinker.visual.merger', 'thinker.visual.merger_list'], vision_tower=['thinker.audio_tower', 'thinker.visual'], generator=['talker', 'code2wav']), architectures=['Qwen3OmniMoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.57.dev0', 'soundfile', 'decord', 'qwen_omni_utils'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3-Omni-30B-A3B-Instruct", + "template_meta": "QwenTemplateMeta(template_type='qwen3_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='\\n', non_thinking_prefix='', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_omni', model_types=['qwen3_omni_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 1, + "num_microbatches": 8 +} \ No newline at end of file