| { |
| "src_vocab_size": 36000, |
| "save_data": "toy-lanotes/data", |
| "src_words_min_frequency": 2, |
| "seed": 1234, |
| "tgt_vocab": "toy-lanotes/vocab.shared", |
| "report_every": 1, |
| "src_vocab": "toy-lanotes/vocab.shared", |
| "tensorboard": true, |
| "tgt_vocab_size": 36000, |
| "n_sample": 0, |
| "valid_metrics": [ |
| "BLEU" |
| ], |
| "vocab_size_multiple": 8, |
| "tgt_words_min_frequency": 2, |
| "overwrite": true, |
| "share_vocab": true, |
| "tensorboard_log_dir": "tensorboard_logging", |
| "tensorboard_log_dir_dated": "tensorboard_logging/Nov-10_14-13-01", |
| "log_file": "train.log", |
| "training": { |
| "world_size": 1, |
| "compute_dtype": "torch.float16", |
| "dropout_steps": [ |
| 0 |
| ], |
| "max_grad_norm": 1.0, |
| "label_smoothing": 0.1, |
| "valid_batch_size": 1024, |
| "save_checkpoint_steps": 1000, |
| "accum_steps": [ |
| 0 |
| ], |
| "average_decay": 0.0, |
| "accum_count": [ |
| 33 |
| ], |
| "decay_method": "noam", |
| "batch_size": 1500, |
| "param_init_method": "xavier_uniform", |
| "learning_rate": 2.0, |
| "use_amp": true, |
| "dropout": [ |
| 0.1 |
| ], |
| "attention_dropout": [ |
| 0.1 |
| ], |
| "batch_size_multiple": 8, |
| "bucket_size": 32768, |
| "normalization": "tokens", |
| "gpu_ranks": [ |
| 0 |
| ], |
| "optim": "adamw", |
| "train_steps": 10000, |
| "adam_beta2": 0.998, |
| "train_from": "toy-lanotes/model", |
| "valid_steps": 100, |
| "prefetch_factor": 50, |
| "warmup_steps": 800, |
| "batch_type": "tokens", |
| "model_path": "toy-lanotes/model", |
| "num_workers": 0, |
| "keep_checkpoint": 50 |
| }, |
| "data": { |
| "corpus_1": { |
| "path_tgt": "toy-lanotes/train.tgt.encoded.filter.shuf", |
| "transforms": [], |
| "path_align": null, |
| "path_src": "toy-lanotes/train.src.encoded.filter.shuf" |
| }, |
| "valid": { |
| "path_tgt": "toy-lanotes/valid.tgt.encoded.filter.shuf", |
| "transforms": [], |
| "path_align": null, |
| "path_src": "toy-lanotes/valid.src.encoded.filter.shuf" |
| } |
| }, |
| "model": { |
| "share_embeddings": true, |
| "transformer_ff": 4096, |
| "architecture": "transformer", |
| "share_decoder_embeddings": true, |
| "position_encoding_type": "SinusoidalInterleaved", |
| "layers": 6, |
| "hidden_size": 1024, |
| "heads": 16, |
| "decoder": { |
| "transformer_ff": 4096, |
| "n_positions": null, |
| "with_cross_attn": true, |
| "decoder_type": "transformer", |
| "tgt_word_vec_size": 1024, |
| "position_encoding_type": "SinusoidalInterleaved", |
| "layers": 6, |
| "hidden_size": 1024, |
| "heads": 16 |
| }, |
| "embeddings": { |
| "src_word_vec_size": 1024, |
| "word_vec_size": 1024, |
| "tgt_word_vec_size": 1024, |
| "position_encoding_type": "SinusoidalInterleaved" |
| }, |
| "encoder": { |
| "src_word_vec_size": 1024, |
| "transformer_ff": 4096, |
| "n_positions": null, |
| "position_encoding_type": "SinusoidalInterleaved", |
| "layers": 6, |
| "encoder_type": "transformer", |
| "hidden_size": 1024, |
| "heads": 16 |
| } |
| } |
| } |
