MultilingualMultiDomain/MultiDomain/preprocess_multilingual/preprocess.log

Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=True, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict=None, suppress_crashes=False, target_lang='si', task='translation', tensorboard_logdir=None, testpref=None, tgtdict=None, threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/en-si/train.bpe', use_plasma_view=False, user_dir=None, validpref='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/en-si/valid.bpe', wandb_project=None, workers=16)
[en] Dictionary: 15936 types
[en] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/en-si/train.bpe.en: 9000 sents, 217040 tokens, 0.0% replaced (by <unk>)
[en] Dictionary: 15936 types
[en] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/en-si/valid.bpe.en: 1008 sents, 26970 tokens, 0.167% replaced (by <unk>)
[si] Dictionary: 15936 types
[si] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/en-si/train.bpe.si: 9000 sents, 208830 tokens, 0.0% replaced (by <unk>)
[si] Dictionary: 15936 types
[si] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/en-si/valid.bpe.si: 1008 sents, 26505 tokens, 0.113% replaced (by <unk>)
Wrote preprocessed data to ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/
Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='si', srcdict='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/dict.en.txt', suppress_crashes=False, target_lang='en', task='translation', tensorboard_logdir=None, testpref=None, tgtdict='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/dict.en.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/si-en/train.bpe', use_plasma_view=False, user_dir=None, validpref='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/si-en/valid.bpe', wandb_project=None, workers=16)
[si] Dictionary: 15936 types
[si] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/si-en/train.bpe.si: 9000 sents, 208830 tokens, 0.0% replaced (by <unk>)
[si] Dictionary: 15936 types
[si] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/si-en/valid.bpe.si: 1008 sents, 26505 tokens, 0.113% replaced (by <unk>)
[en] Dictionary: 15936 types
[en] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/si-en/train.bpe.en: 9000 sents, 217040 tokens, 0.0% replaced (by <unk>)
[en] Dictionary: 15936 types
[en] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/si-en/valid.bpe.en: 1008 sents, 26970 tokens, 0.167% replaced (by <unk>)
Wrote preprocessed data to ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/
Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/dict.en.txt', suppress_crashes=False, target_lang='ta', task='translation', tensorboard_logdir=None, testpref=None, tgtdict='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/dict.en.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/en-ta/train.bpe', use_plasma_view=False, user_dir=None, validpref='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/en-ta/valid.bpe', wandb_project=None, workers=16)
[en] Dictionary: 15936 types
[en] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/en-ta/train.bpe.en: 9000 sents, 227342 tokens, 1.06% replaced (by <unk>)
[en] Dictionary: 15936 types
[en] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/en-ta/valid.bpe.en: 1008 sents, 28196 tokens, 1.11% replaced (by <unk>)
[ta] Dictionary: 15936 types
[ta] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/en-ta/train.bpe.ta: 9000 sents, 215442 tokens, 82.7% replaced (by <unk>)
[ta] Dictionary: 15936 types
[ta] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/en-ta/valid.bpe.ta: 1008 sents, 27050 tokens, 83.5% replaced (by <unk>)
Wrote preprocessed data to ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/
Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='ta', srcdict='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/dict.en.txt', suppress_crashes=False, target_lang='en', task='translation', tensorboard_logdir=None, testpref=None, tgtdict='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/dict.en.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/ta-en/train.bpe', use_plasma_view=False, user_dir=None, validpref='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/ta-en/valid.bpe', wandb_project=None, workers=16)
[ta] Dictionary: 15936 types
[ta] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/ta-en/train.bpe.ta: 9000 sents, 215442 tokens, 82.7% replaced (by <unk>)
[ta] Dictionary: 15936 types
[ta] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/ta-en/valid.bpe.ta: 1008 sents, 27050 tokens, 83.5% replaced (by <unk>)
[en] Dictionary: 15936 types
[en] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/ta-en/train.bpe.en: 9000 sents, 227342 tokens, 1.06% replaced (by <unk>)
[en] Dictionary: 15936 types
[en] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/ta-en/valid.bpe.en: 1008 sents, 28196 tokens, 1.11% replaced (by <unk>)
Wrote preprocessed data to ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/
Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='si', srcdict='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/dict.en.txt', suppress_crashes=False, target_lang='ta', task='translation', tensorboard_logdir=None, testpref=None, tgtdict='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/dict.en.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/si-ta/train.bpe', use_plasma_view=False, user_dir=None, validpref='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/si-ta/valid.bpe', wandb_project=None, workers=16)
[si] Dictionary: 15936 types
[si] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/si-ta/train.bpe.si: 6509 sents, 131101 tokens, 0.102% replaced (by <unk>)
[si] Dictionary: 15936 types
[si] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/si-ta/valid.bpe.si: 560 sents, 14222 tokens, 0.0914% replaced (by <unk>)
[ta] Dictionary: 15936 types
[ta] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/si-ta/train.bpe.ta: 6509 sents, 127998 tokens, 82.0% replaced (by <unk>)
[ta] Dictionary: 15936 types
[ta] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/si-ta/valid.bpe.ta: 560 sents, 14268 tokens, 84.9% replaced (by <unk>)
Wrote preprocessed data to ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/
Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='ta', srcdict='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/dict.en.txt', suppress_crashes=False, target_lang='si', task='translation', tensorboard_logdir=None, testpref=None, tgtdict='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/dict.en.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/ta-si/train.bpe', use_plasma_view=False, user_dir=None, validpref='../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/ta-si/valid.bpe', wandb_project=None, workers=16)
[ta] Dictionary: 15936 types
[ta] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/ta-si/train.bpe.ta: 6509 sents, 127998 tokens, 82.0% replaced (by <unk>)
[ta] Dictionary: 15936 types
[ta] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/ta-si/valid.bpe.ta: 560 sents, 14268 tokens, 84.9% replaced (by <unk>)
[si] Dictionary: 15936 types
[si] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/ta-si/train.bpe.si: 6509 sents, 131101 tokens, 0.102% replaced (by <unk>)
[si] Dictionary: 15936 types
[si] ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/encoded/ta-si/valid.bpe.si: 560 sents, 14222 tokens, 0.0914% replaced (by <unk>)
Wrote preprocessed data to ../trained_models/Fairseq/MultilingualMultiDomain/MultiDomain/preprocess_multilingual/