| { |
| "name": "C2", |
| "description": "RoPE instead of sinusoidal absolute", |
| "model": { |
| "dim_model": 128, |
| "num_heads": 4, |
| "num_encoder_layers": 2, |
| "num_decoder_layers": 2, |
| "dim_ff": 512, |
| "dropout": 0.1, |
| "norm_eps": 1e-06, |
| "num_kv_heads": 2, |
| "rope_base": 10000.0, |
| "pos_encoding": "rope", |
| "attention": "mha", |
| "norm": "layernorm", |
| "tokenization": "subword", |
| "tie_output_embedding": false |
| }, |
| "data": { |
| "cipher_path": "/home2/winterdew/ANLP_A1/ANLP-M26-Assignment-1/dataset/brown_cipher.txt", |
| "plain_path": "/home2/winterdew/ANLP_A1/ANLP-M26-Assignment-1/dataset/brown_plain.txt", |
| "split_ratios": [ |
| 0.8, |
| 0.1, |
| 0.1 |
| ], |
| "split_seed": 1234, |
| "max_src_len_subword": 768, |
| "max_tgt_len_subword": 256, |
| "max_src_len_byte": 3072, |
| "max_tgt_len_byte": 1152, |
| "length_policy": "drop", |
| "effective_batch_size": 64, |
| "micro_batch_size_subword": 64, |
| "micro_batch_size_byte": 64, |
| "num_workers": 0, |
| "verbose": true, |
| "tokenizer": { |
| "src_vocab_size": 4096, |
| "tgt_vocab_size": 4096, |
| "src_seed_size": 20000, |
| "tgt_seed_size": 50000, |
| "src_max_segment": 16, |
| "tgt_max_segment": 32, |
| "max_piece_len": 16, |
| "shrink_factor": 0.75, |
| "em_iters": 2, |
| "min_freq": 2, |
| "sample_seed": 1234, |
| "cache_dir": "/home2/winterdew/ANLP_A1/ANLP-M26-Assignment-1/outputs/tokenizers", |
| "retrain": false |
| }, |
| "window": { |
| "enabled": true, |
| "src_window": 1024, |
| "src_stride": 768, |
| "min_tgt_chars": 16, |
| "eval_full_sentences": true |
| } |
| }, |
| "train": { |
| "epochs": 100, |
| "lr": 0.0007, |
| "betas": [ |
| 0.9, |
| 0.98 |
| ], |
| "adam_eps": 1e-09, |
| "weight_decay": 0.01, |
| "grad_clip": 1.0, |
| "label_smoothing": 0.05, |
| "scheduler": "cosine", |
| "warmup_steps": 400, |
| "min_lr": 1e-05, |
| "seed": 26, |
| "device": "auto", |
| "amp": true, |
| "eval_every_epochs": 1, |
| "generate_every_epochs": 5, |
| "max_eval_batches": null, |
| "length_slack": 16, |
| "gen_eval_examples": 128, |
| "gen_batch_size": 32, |
| "test_gen_examples": null, |
| "log_every_steps": 50, |
| "checkpoint_dir": "/home2/winterdew/ANLP_A1/ANLP-M26-Assignment-1/outputs/checkpoints", |
| "keep_best_only": true, |
| "early_stopping_patience": null, |
| "wandb_project": "anlp-a1-transformers", |
| "wandb_entity": null, |
| "wandb_mode": "online" |
| }, |
| "blt": { |
| "patch_size": 8, |
| "local_dim": 128, |
| "local_heads": 4, |
| "local_dim_ff": 512, |
| "local_encoder_layers": 1, |
| "local_decoder_layers": 1, |
| "local_dropout": 0.1 |
| }, |
| "derived": { |
| "token_free": false, |
| "max_src_len": 1026, |
| "max_tgt_len": 1026, |
| "micro_batch_size": 64, |
| "windowed": true, |
| "grad_accum_steps": 1, |
| "pe_max_len": 1042, |
| "head_dim": 32 |
| } |
| } |
