combilatent / src /xp1 /3_run.py
younadi's picture
Add files using upload-large-folder tool
4417138 verified
Raw History Blame Contribute Delete
3.99 kB
from src.train import train
import tqdm
params = [
{
"testing": False,
"seed": 97,
"data_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}",
"n_embd": n_embd,
"n_head": n_head,
"n_layer": n_layer,
"ff_width": 4,
"intermediate_schedules": True,
"train_batch_size": 512,
"val_batch_size": 256,
"nb_epochs": 5,
"early_stopping_patience": 15,
"dropout": 0.0,
"checkpoint_interval_ratio": 1.0,
"decay_lr": True,
"lr_partitions_ratios": [0.66],
"init_lr": 1e-4,
"max_lr": 1e-3,
"min_lr": 5e-5,
"lr_warmup_iters_ratio": 0.1,
"lr_decay_iters_ratio": 0.95,
"beta1": 0.9,
"beta2": 0.95,
"weight_decay": wd,
"grad_clip": 1.0,
"compile": "",
"compile_mode": "default",
"save_only_last_checkpoint": True,
"output_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}/train_nEmbd{n_embd}_nHead{n_head}_nLayer{n_layer}",
}
for nb_jobs in [9]
for nb_machines in [6]
for top_k in [0]
for n_embd in [6, 8, 12, 24, 36, 48, 64]
for n_head in [1, 2]
for n_layer in [1, 2, 3, 4]
for wd in [10e+0]
]
for param in (pbar:=tqdm.tqdm(params)):
pbar.set_description(f"Section 1. Phase 1 Configuration: n_embd={param['n_embd']}, n_head={param['n_head']}, n_layer={param['n_layer']}")
train(**param)
params = [
{
"testing": False,
"seed": 97,
"data_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}",
"n_embd": n_embd,
"n_head": n_head,
"n_layer": n_layer,
"ff_width": 4,
"intermediate_schedules": True,
"train_batch_size": 512,
"val_batch_size": 256,
"nb_epochs": 5,
"early_stopping_patience": 15,
"dropout": 0.0,
"checkpoint_interval_ratio": 1.0,
"decay_lr": True,
"lr_partitions_ratios": [0.66],
"init_lr": 1e-4,
"max_lr": 1e-3,
"min_lr": 5e-5,
"lr_warmup_iters_ratio": 0.1,
"lr_decay_iters_ratio": 0.95,
"beta1": 0.9,
"beta2": 0.95,
"weight_decay": wd,
"grad_clip": 1.0,
"compile": "",
"compile_mode": "default",
"save_only_last_checkpoint": True,
"output_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}/train_nEmbd{n_embd}_nHead{n_head}_nLayer{n_layer}",
}
for nb_jobs in [9]
for nb_machines in [6]
for top_k in [0]
for n_embd in [6 + 3*i for i in range(0, 21)]
for n_head in [3]
for n_layer in [1, 2, 3, 4]
for wd in [10e+0]
]
for param in (pbar:=tqdm.tqdm(params)):
pbar.set_description(f"Section 2. Phase 1 Configuration: n_embd={param['n_embd']}, n_head={param['n_head']}, n_layer={param['n_layer']}")
train(**param)
params = [
{
"testing": False,
"seed": 97,
"data_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}",
"n_embd": n_embd,
"n_head": n_head,
"n_layer": n_layer,
"ff_width": 4,
"intermediate_schedules": True,
"train_batch_size": 512,
"val_batch_size": 256,
"nb_epochs": 5,
"early_stopping_patience": 15,
"dropout": 0.0,
"checkpoint_interval_ratio": 1.0,
"decay_lr": True,
"lr_partitions_ratios": [0.66],
"init_lr": 1e-4,
"max_lr": 1e-3,
"min_lr": 5e-5,
"lr_warmup_iters_ratio": 0.1,
"lr_decay_iters_ratio": 0.95,
"beta1": 0.9,
"beta2": 0.95,
"weight_decay": wd,
"grad_clip": 1.0,
"compile": "",
"compile_mode": "default",
"save_only_last_checkpoint": True,
"output_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}/train_nEmbd{n_embd}_nHead{n_head}_nLayer{n_layer}",
}
for nb_jobs in [9]
for nb_machines in [6]
for top_k in [0]
for n_embd in [8 + 4*i for i in range(0, 15)]
for n_head in [4]
for n_layer in [1, 2, 3, 4]
for wd in [10e+0]
]
for param in (pbar:=tqdm.tqdm(params)):
pbar.set_description(f"Section 3. Phase 1 Configuration: n_embd={param['n_embd']}, n_head={param['n_head']}, n_layer={param['n_layer']}")
train(**param)