Download src/xp0/source/launch_train.py from Albidar/combilatent: direct link, hf CLI and curl.
- Browser
- Download file 1.33 kB
-
https://huggingface.co/Albidar/combilatent/resolve/main/src/xp0/source/launch_train.py
- Command line
-
hf download hf://Albidar/combilatent/src/xp0/source/launch_train.py
-
curl -L -o launch_train.py https://huggingface.co/Albidar/combilatent/resolve/main/src/xp0/source/launch_train.py
1.33 kB
| from train import train | |
| params = [ | |
| { | |
| "testing": False, | |
| "seed": 97, | |
| "data_dir": f"../datasets/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}", | |
| "n_embd": n_embd, | |
| "n_head": n_head, | |
| "n_layer": n_layer, | |
| "ff_width": 4, | |
| "intermediate_schedules": True, | |
| "train_batch_size": 512, | |
| "val_batch_size": 256, | |
| "nb_epochs": 5, | |
| "early_stopping_patience": 15, | |
| "dropout": 0.0, | |
| "checkpoint_interval_ratio": 1.0, | |
| "decay_lr": True, | |
| "lr_partitions_ratios": [0.66], | |
| "init_lr": 1e-4, | |
| "max_lr": 1e-3, | |
| "min_lr": 5e-5, | |
| "lr_warmup_iters_ratio": 0.1, | |
| "lr_decay_iters_ratio": 0.95, | |
| "beta1": 0.9, | |
| "beta2": 0.95, | |
| "weight_decay": wd, | |
| "grad_clip": 1.0, | |
| "compile": "", | |
| "compile_mode": "default", | |
| "save_only_last_checkpoint": True, | |
| "output_dir": f"../datasets/exhaustive_{nb_jobs}_{nb_machines}/top_{top_k}/train_{model_size_code}_Wd{wd_str}", | |
| } | |
| for nb_jobs in [7, 8, 9] | |
| for nb_machines in [2, 3, 4, 5, 6] | |
| for top_k in [0, 1, 2, 3, 4] | |
| for model_size_code, n_embd, n_head, n_layer in [("Bm", 256, 16, 8), ("Mm", 128, 8, 4), ("Sm", 64, 4, 2)] # Bm: Big model, Mm: Medium model, Sm: Small model | |
| for wd, wd_str in [(1e-1, "1e-1"), (1e-0, "1e-0"), (5e-0, "5e-0"), (10e-0, "10e-0")] | |
| ] | |
| for param in params: | |
| train(**param) |