Download src/xp1/3_run.py from Albidar/combilatent: direct link, hf CLI and curl.
- Browser
- Download file 3.99 kB
-
https://huggingface.co/Albidar/combilatent/resolve/main/src/xp1/3_run.py
- Command line
-
hf download hf://Albidar/combilatent/src/xp1/3_run.py
-
curl -L -o 3_run.py https://huggingface.co/Albidar/combilatent/resolve/main/src/xp1/3_run.py
3.99 kB
| from src.train import train | |
| import tqdm | |
| params = [ | |
| { | |
| "testing": False, | |
| "seed": 97, | |
| "data_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}", | |
| "n_embd": n_embd, | |
| "n_head": n_head, | |
| "n_layer": n_layer, | |
| "ff_width": 4, | |
| "intermediate_schedules": True, | |
| "train_batch_size": 512, | |
| "val_batch_size": 256, | |
| "nb_epochs": 5, | |
| "early_stopping_patience": 15, | |
| "dropout": 0.0, | |
| "checkpoint_interval_ratio": 1.0, | |
| "decay_lr": True, | |
| "lr_partitions_ratios": [0.66], | |
| "init_lr": 1e-4, | |
| "max_lr": 1e-3, | |
| "min_lr": 5e-5, | |
| "lr_warmup_iters_ratio": 0.1, | |
| "lr_decay_iters_ratio": 0.95, | |
| "beta1": 0.9, | |
| "beta2": 0.95, | |
| "weight_decay": wd, | |
| "grad_clip": 1.0, | |
| "compile": "", | |
| "compile_mode": "default", | |
| "save_only_last_checkpoint": True, | |
| "output_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}/train_nEmbd{n_embd}_nHead{n_head}_nLayer{n_layer}", | |
| } | |
| for nb_jobs in [9] | |
| for nb_machines in [6] | |
| for top_k in [0] | |
| for n_embd in [6, 8, 12, 24, 36, 48, 64] | |
| for n_head in [1, 2] | |
| for n_layer in [1, 2, 3, 4] | |
| for wd in [10e+0] | |
| ] | |
| for param in (pbar:=tqdm.tqdm(params)): | |
| pbar.set_description(f"Section 1. Phase 1 Configuration: n_embd={param['n_embd']}, n_head={param['n_head']}, n_layer={param['n_layer']}") | |
| train(**param) | |
| params = [ | |
| { | |
| "testing": False, | |
| "seed": 97, | |
| "data_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}", | |
| "n_embd": n_embd, | |
| "n_head": n_head, | |
| "n_layer": n_layer, | |
| "ff_width": 4, | |
| "intermediate_schedules": True, | |
| "train_batch_size": 512, | |
| "val_batch_size": 256, | |
| "nb_epochs": 5, | |
| "early_stopping_patience": 15, | |
| "dropout": 0.0, | |
| "checkpoint_interval_ratio": 1.0, | |
| "decay_lr": True, | |
| "lr_partitions_ratios": [0.66], | |
| "init_lr": 1e-4, | |
| "max_lr": 1e-3, | |
| "min_lr": 5e-5, | |
| "lr_warmup_iters_ratio": 0.1, | |
| "lr_decay_iters_ratio": 0.95, | |
| "beta1": 0.9, | |
| "beta2": 0.95, | |
| "weight_decay": wd, | |
| "grad_clip": 1.0, | |
| "compile": "", | |
| "compile_mode": "default", | |
| "save_only_last_checkpoint": True, | |
| "output_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}/train_nEmbd{n_embd}_nHead{n_head}_nLayer{n_layer}", | |
| } | |
| for nb_jobs in [9] | |
| for nb_machines in [6] | |
| for top_k in [0] | |
| for n_embd in [6 + 3*i for i in range(0, 21)] | |
| for n_head in [3] | |
| for n_layer in [1, 2, 3, 4] | |
| for wd in [10e+0] | |
| ] | |
| for param in (pbar:=tqdm.tqdm(params)): | |
| pbar.set_description(f"Section 2. Phase 1 Configuration: n_embd={param['n_embd']}, n_head={param['n_head']}, n_layer={param['n_layer']}") | |
| train(**param) | |
| params = [ | |
| { | |
| "testing": False, | |
| "seed": 97, | |
| "data_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}", | |
| "n_embd": n_embd, | |
| "n_head": n_head, | |
| "n_layer": n_layer, | |
| "ff_width": 4, | |
| "intermediate_schedules": True, | |
| "train_batch_size": 512, | |
| "val_batch_size": 256, | |
| "nb_epochs": 5, | |
| "early_stopping_patience": 15, | |
| "dropout": 0.0, | |
| "checkpoint_interval_ratio": 1.0, | |
| "decay_lr": True, | |
| "lr_partitions_ratios": [0.66], | |
| "init_lr": 1e-4, | |
| "max_lr": 1e-3, | |
| "min_lr": 5e-5, | |
| "lr_warmup_iters_ratio": 0.1, | |
| "lr_decay_iters_ratio": 0.95, | |
| "beta1": 0.9, | |
| "beta2": 0.95, | |
| "weight_decay": wd, | |
| "grad_clip": 1.0, | |
| "compile": "", | |
| "compile_mode": "default", | |
| "save_only_last_checkpoint": True, | |
| "output_dir": f"outputs/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}/train_nEmbd{n_embd}_nHead{n_head}_nLayer{n_layer}", | |
| } | |
| for nb_jobs in [9] | |
| for nb_machines in [6] | |
| for top_k in [0] | |
| for n_embd in [8 + 4*i for i in range(0, 15)] | |
| for n_head in [4] | |
| for n_layer in [1, 2, 3, 4] | |
| for wd in [10e+0] | |
| ] | |
| for param in (pbar:=tqdm.tqdm(params)): | |
| pbar.set_description(f"Section 3. Phase 1 Configuration: n_embd={param['n_embd']}, n_head={param['n_head']}, n_layer={param['n_layer']}") | |
| train(**param) |