File size: 2,550 Bytes
0ab2566 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 | # CODI + PMPS Training Configuration for GSM8k
# Based on codi_ori/args/gsm_codi_train.yaml with PMPS-specific additions
model_name_or_path: /data/oss_bucket_0/liqi/models/gpt2
lora_r: 128
lora_alpha: 32
lora_init: True
use_lora: True
do_train: True
# ==================== Data Arguments ====================
data_name: icot
# data_name: icot-full
# ==================== Training Arguments ====================
output_dir: /data/oss_bucket_0/liqi/codi_pmps/c2l_k24_t0.1_mt0.1_fac0.5_l6_lr3e3_btz256_pos2_cosine_ep50
expt_name: gsm8k_gpt_pmps
logging_dir: /data/oss_bucket_0/liqi/codi_pmps/c2l_k24_t0.1_mt0.1_fac0.5_l6_lr3e3_btz256_pos2_cosine_ep50/logs
logging_steps: 10
logging_strategy: steps
enable_top_k: False
latent_top_k: 0
latent_top_k_temperature: 1.0
# ===== PMPS Loss (NEW) =====
pmps_loss_factor: 0.5
pmps_temperature: 0.1
pmps_match_temperature: 0.1
pmps_epsilon: 0.05
pmps_sinkhorn_iterations: 3
pmps_nmb_prototypes: 24 # ~4 × num_latent (num_latent=6 -> K=24)
pmps_feat_dim: 128
pmps_hidden_mlp: 512
pmps_use_projector: True
pmps_freeze_prototypes_steps: 0
pmps_layer: last
pmps_start_epoch: 0
# ===== Progressive Sequential PMPS (NEW) =====
pmps_position_bias_max: 2.0 # Max positional bias strength. 0=disabled. Recommended: 3.0~8.0
pmps_position_bias_schedule: constant # Decay schedule: cosine / linear / constant
# ===== Latent Diversity Regularisation (NEW) =====
diversity_loss_factor: 0.0 # Weight for diversity loss. 0=disabled. Recommended: 0.05~0.3
seed: 42
model_max_length: 512
per_device_train_batch_size: 32
# per_device_train_batch_size: 32
# per_device_train_batch_size: 8
# gradient_accumulation_steps: 2
gradient_accumulation_steps: 1
bf16: True
num_train_epochs: 50
# learning_rate: !!float 3e-3
learning_rate: !!float 3e-3
max_grad_norm: 2.0
save_strategy: "epoch"
save_safetensors: False
save_total_limit: 50
weight_decay: 0.1
warmup_ratio: 0.03
lr_scheduler_type: cosine
report_to: wandb
include_last_cot: False
# ==================== Wandb Arguments ====================
use_wandb: True
wandb_api_key: "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" # Fill in your wandb API key here
wandb_project: codi-pmps
wandb_run_name: c2l_k24_t0.1_mt0.1_fac0.5_l6_lr3e3_btz256_pos2_cosine_ep50
# ==================== CODI Specific Arguments ====================
num_latent: 6
use_prj: True
prj_dim: 768
prj_dropout: 0.0
distill_loss_div_std: True
exp_mode: False
exp_data_num: 2000
remove_eos: True
print_ref_model_stats: True
distill_loss_factor: 1.0
|