from src.train import train import tqdm params = [ { "testing": False, "seed": 97, "data_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}", "n_embd": n_embd, "n_head": n_head, "n_layer": n_layer, "ff_width": 4, "intermediate_schedules": True, "train_batch_size": 512, "val_batch_size": 256, "nb_epochs": 5, "early_stopping_patience": 15, "dropout": 0.0, "checkpoint_interval_ratio": 1.0, "decay_lr": True, "lr_partitions_ratios": [0.66], "init_lr": 1e-4, "max_lr": 1e-3, "min_lr": 5e-5, "lr_warmup_iters_ratio": 0.1, "lr_decay_iters_ratio": 0.95, "beta1": 0.9, "beta2": 0.95, "weight_decay": wd, "grad_clip": 1.0, "compile": "", "compile_mode": "default", "save_only_last_checkpoint": True, "output_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}/train_nEmbd{n_embd}_nHead{n_head}_nLayer{n_layer}", } for nb_jobs in [9] for nb_machines in [6] for top_k in [0] for n_embd in [6, 8, 12, 24, 36, 48, 64] for n_head in [1, 2] for n_layer in [1, 2, 3, 4] for wd in [10e+0] ] for param in (pbar:=tqdm.tqdm(params)): pbar.set_description(f"Section 1. Phase 1 Configuration: n_embd={param['n_embd']}, n_head={param['n_head']}, n_layer={param['n_layer']}") train(**param) params = [ { "testing": False, "seed": 97, "data_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}", "n_embd": n_embd, "n_head": n_head, "n_layer": n_layer, "ff_width": 4, "intermediate_schedules": True, "train_batch_size": 512, "val_batch_size": 256, "nb_epochs": 5, "early_stopping_patience": 15, "dropout": 0.0, "checkpoint_interval_ratio": 1.0, "decay_lr": True, "lr_partitions_ratios": [0.66], "init_lr": 1e-4, "max_lr": 1e-3, "min_lr": 5e-5, "lr_warmup_iters_ratio": 0.1, "lr_decay_iters_ratio": 0.95, "beta1": 0.9, "beta2": 0.95, "weight_decay": wd, "grad_clip": 1.0, "compile": "", "compile_mode": "default", "save_only_last_checkpoint": True, "output_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}/train_nEmbd{n_embd}_nHead{n_head}_nLayer{n_layer}", } for nb_jobs in [9] for nb_machines in [6] for top_k in [0] for n_embd in [6 + 3*i for i in range(0, 21)] for n_head in [3] for n_layer in [1, 2, 3, 4] for wd in [10e+0] ] for param in (pbar:=tqdm.tqdm(params)): pbar.set_description(f"Section 2. Phase 1 Configuration: n_embd={param['n_embd']}, n_head={param['n_head']}, n_layer={param['n_layer']}") train(**param) params = [ { "testing": False, "seed": 97, "data_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}", "n_embd": n_embd, "n_head": n_head, "n_layer": n_layer, "ff_width": 4, "intermediate_schedules": True, "train_batch_size": 512, "val_batch_size": 256, "nb_epochs": 5, "early_stopping_patience": 15, "dropout": 0.0, "checkpoint_interval_ratio": 1.0, "decay_lr": True, "lr_partitions_ratios": [0.66], "init_lr": 1e-4, "max_lr": 1e-3, "min_lr": 5e-5, "lr_warmup_iters_ratio": 0.1, "lr_decay_iters_ratio": 0.95, "beta1": 0.9, "beta2": 0.95, "weight_decay": wd, "grad_clip": 1.0, "compile": "", "compile_mode": "default", "save_only_last_checkpoint": True, "output_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}/train_nEmbd{n_embd}_nHead{n_head}_nLayer{n_layer}", } for nb_jobs in [9] for nb_machines in [6] for top_k in [0] for n_embd in [8 + 4*i for i in range(0, 15)] for n_head in [4] for n_layer in [1, 2, 3, 4] for wd in [10e+0] ] for param in (pbar:=tqdm.tqdm(params)): pbar.set_description(f"Section 3. Phase 1 Configuration: n_embd={param['n_embd']}, n_head={param['n_head']}, n_layer={param['n_layer']}") train(**param)