Training in progress, epoch 1
Browse files- config.json +2 -1
- model.safetensors +1 -1
- tokenizer_config.json +0 -1
- training_args.bin +1 -1
- wandb/debug-internal.log +7 -0
- wandb/debug.log +25 -0
- wandb/run-20250520_132739-ezfnx1ko/files/output.log +32 -0
- wandb/run-20250520_132739-ezfnx1ko/files/requirements.txt +117 -0
- wandb/run-20250520_132739-ezfnx1ko/files/wandb-metadata.json +42 -0
- wandb/run-20250520_132739-ezfnx1ko/logs/debug-core.log +6 -0
- wandb/run-20250520_132739-ezfnx1ko/logs/debug-internal.log +7 -0
- wandb/run-20250520_132739-ezfnx1ko/logs/debug.log +25 -0
- wandb/run-20250520_132739-ezfnx1ko/run-ezfnx1ko.wandb +0 -0
config.json
CHANGED
|
@@ -1,4 +1,5 @@
|
|
| 1 |
{
|
|
|
|
| 2 |
"activation_dropout": 0.0,
|
| 3 |
"activation_function": "relu",
|
| 4 |
"architectures": [
|
|
@@ -28,7 +29,7 @@
|
|
| 28 |
"scale_embedding": true,
|
| 29 |
"tokenizer_class": "NllbTokenizer",
|
| 30 |
"torch_dtype": "float32",
|
| 31 |
-
"transformers_version": "4.
|
| 32 |
"use_cache": true,
|
| 33 |
"vocab_size": 256206
|
| 34 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"_name_or_path": "facebook/nllb-200-distilled-600M",
|
| 3 |
"activation_dropout": 0.0,
|
| 4 |
"activation_function": "relu",
|
| 5 |
"architectures": [
|
|
|
|
| 29 |
"scale_embedding": true,
|
| 30 |
"tokenizer_class": "NllbTokenizer",
|
| 31 |
"torch_dtype": "float32",
|
| 32 |
+
"transformers_version": "4.46.3",
|
| 33 |
"use_cache": true,
|
| 34 |
"vocab_size": 256206
|
| 35 |
}
|
model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 2460354912
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f691a3e6dc736a73cf8c38dd485dece2c7f9686abb5a46e33222e609b15618d9
|
| 3 |
size 2460354912
|
tokenizer_config.json
CHANGED
|
@@ -1865,7 +1865,6 @@
|
|
| 1865 |
"clean_up_tokenization_spaces": false,
|
| 1866 |
"cls_token": "<s>",
|
| 1867 |
"eos_token": "</s>",
|
| 1868 |
-
"extra_special_tokens": {},
|
| 1869 |
"legacy_behaviour": false,
|
| 1870 |
"mask_token": "<mask>",
|
| 1871 |
"model_max_length": 1024,
|
|
|
|
| 1865 |
"clean_up_tokenization_spaces": false,
|
| 1866 |
"cls_token": "<s>",
|
| 1867 |
"eos_token": "</s>",
|
|
|
|
| 1868 |
"legacy_behaviour": false,
|
| 1869 |
"mask_token": "<mask>",
|
| 1870 |
"model_max_length": 1024,
|
training_args.bin
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 5905
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ed64f35a061728dc4646dc67331f716c8f82edd151b9ff2a8baf27590dd7674b
|
| 3 |
size 5905
|
wandb/debug-internal.log
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2025-05-20T13:27:39.715003286Z","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"models/wandb/run-20250520_132739-ezfnx1ko/logs/debug-core.log"}
|
| 2 |
+
{"time":"2025-05-20T13:27:39.858492158Z","level":"INFO","msg":"created new stream","id":"ezfnx1ko"}
|
| 3 |
+
{"time":"2025-05-20T13:27:39.858583385Z","level":"INFO","msg":"stream: started","id":"ezfnx1ko"}
|
| 4 |
+
{"time":"2025-05-20T13:27:39.858611644Z","level":"INFO","msg":"writer: Do: started","stream_id":"ezfnx1ko"}
|
| 5 |
+
{"time":"2025-05-20T13:27:39.858799027Z","level":"INFO","msg":"sender: started","stream_id":"ezfnx1ko"}
|
| 6 |
+
{"time":"2025-05-20T13:27:39.858821988Z","level":"INFO","msg":"handler: started","stream_id":"ezfnx1ko"}
|
| 7 |
+
{"time":"2025-05-20T13:27:39.960784281Z","level":"INFO","msg":"Starting system monitor"}
|
wandb/debug.log
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
|
| 2 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_setup.py:_flush():70] Configure stats pid to 14446
|
| 3 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_setup.py:_flush():70] Loading settings from /home/kolade_esmglobalconsulting_com/.config/wandb/settings
|
| 4 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_setup.py:_flush():70] Loading settings from /home/kolade_esmglobalconsulting_com/Documents/mlops/app/wandb/settings
|
| 5 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_setup.py:_flush():70] Loading settings from environment variables
|
| 6 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_init.py:setup_run_log_directory():724] Logging user logs to models/wandb/run-20250520_132739-ezfnx1ko/logs/debug.log
|
| 7 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to models/wandb/run-20250520_132739-ezfnx1ko/logs/debug-internal.log
|
| 8 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_init.py:init():852] calling init triggers
|
| 9 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
|
| 10 |
+
config: {'_wandb': {}}
|
| 11 |
+
2025-05-20 13:27:39,703 INFO MainThread:14446 [wandb_init.py:init():893] starting backend
|
| 12 |
+
2025-05-20 13:27:39,703 INFO MainThread:14446 [wandb_init.py:init():897] sending inform_init request
|
| 13 |
+
2025-05-20 13:27:39,708 INFO MainThread:14446 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
|
| 14 |
+
2025-05-20 13:27:39,708 INFO MainThread:14446 [wandb_init.py:init():907] backend started and connected
|
| 15 |
+
2025-05-20 13:27:39,710 INFO MainThread:14446 [wandb_init.py:init():1005] updated telemetry
|
| 16 |
+
2025-05-20 13:27:39,710 INFO MainThread:14446 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
|
| 17 |
+
2025-05-20 13:27:39,957 INFO MainThread:14446 [wandb_init.py:init():1104] starting run threads in backend
|
| 18 |
+
2025-05-20 13:27:40,144 INFO MainThread:14446 [wandb_run.py:_console_start():2573] atexit reg
|
| 19 |
+
2025-05-20 13:27:40,145 INFO MainThread:14446 [wandb_run.py:_redirect():2421] redirect: wrap_raw
|
| 20 |
+
2025-05-20 13:27:40,145 INFO MainThread:14446 [wandb_run.py:_redirect():2490] Wrapping output streams.
|
| 21 |
+
2025-05-20 13:27:40,145 INFO MainThread:14446 [wandb_run.py:_redirect():2513] Redirects installed.
|
| 22 |
+
2025-05-20 13:27:40,147 INFO MainThread:14446 [wandb_init.py:init():1150] run started, returning control to user process
|
| 23 |
+
2025-05-20 13:27:40,339 INFO MainThread:14446 [wandb_run.py:_config_callback():1436] config_cb None None {'vocab_size': 256206, 'max_position_embeddings': 1024, 'd_model': 1024, 'encoder_ffn_dim': 4096, 'encoder_layers': 12, 'encoder_attention_heads': 16, 'decoder_ffn_dim': 4096, 'decoder_layers': 12, 'decoder_attention_heads': 16, 'dropout': 0.1, 'attention_dropout': 0.1, 'activation_dropout': 0.0, 'activation_function': 'relu', 'init_std': 0.02, 'encoder_layerdrop': 0, 'decoder_layerdrop': 0, 'use_cache': True, 'num_hidden_layers': 12, 'scale_embedding': True, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'float32', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': True, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 200, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['M2M100ForConditionalGeneration'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': 'NllbTokenizer', 'prefix': None, 'bos_token_id': 0, 'pad_token_id': 1, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': 2, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'facebook/nllb-200-distilled-600M', '_attn_implementation_autoset': True, 'transformers_version': '4.46.3', 'model_type': 'm2m_100', 'output_dir': 'models', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'epoch', 'prediction_loss_only': False, 'per_device_train_batch_size': 16, 'per_device_eval_batch_size': 16, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 2e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': 'models/logs', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 500, 'logging_nan_inf_filter': True, 'save_strategy': 'epoch', 'save_steps': 500, 'save_total_limit': 3, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': 'latest_yoruba_mod', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False, 'average_tokens_across_devices': False, 'sortish_sampler': False, 'predict_with_generate': True, 'generation_max_length': None, 'generation_num_beams': None, 'generation_config': None}
|
| 24 |
+
2025-05-20 13:27:40,341 INFO MainThread:14446 [wandb_config.py:__setitem__():154] [no run ID] config set model/num_parameters = 615073792 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7f78cfa89290>>
|
| 25 |
+
2025-05-20 13:27:40,341 INFO MainThread:14446 [wandb_run.py:_config_callback():1436] config_cb model/num_parameters 615073792 None
|
wandb/run-20250520_132739-ezfnx1ko/files/output.log
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
33%|██████████████████████████████████████ | 6/18 [00:02<00:02, 4.43it/s]Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 2 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 3 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 4 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 5 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 6 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 7 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 8 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 9 |
+
33%|██████████████████████████████████████ | 6/18 [00:03<00:02, 4.43it/s]/home/kolade_esmglobalconsulting_com/Documents/mlops/mlops/lib/python3.11/site-packages/transformers/modeling_utils.py:2817: UserWarning: Moving the following attributes in the config to the generation config: {'max_length': 200}. You are seeing this warning because you've set generation parameters in the model config, as opposed to in the generation config.
|
| 10 |
+
warnings.warn(
|
| 11 |
+
{'eval_loss': 3.410855531692505, 'eval_bleu': 3.2658, 'eval_gen_len': 200.0, 'eval_runtime': 0.8867, 'eval_samples_per_second': 5.639, 'eval_steps_per_second': 1.128, 'epoch': 1.0}
|
| 12 |
+
67%|███████████████████████████████████████████████████████████████████████████▎ | 12/18 [00:16<00:04, 1.21it/s]Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 13 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 14 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 15 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 16 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 17 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 18 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 19 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 20 |
+
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18/18 [00:27<00:00, 1.31it/s]Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 21 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 22 |
+
{'eval_loss': 3.1785385608673096, 'eval_bleu': 3.204, 'eval_gen_len': 200.0, 'eval_runtime': 0.6222, 'eval_samples_per_second': 8.037, 'eval_steps_per_second': 1.607, 'epoch': 2.0}
|
| 23 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 24 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 25 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 26 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 27 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 28 |
+
Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
| 29 |
+
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18/18 [00:59<00:00, 3.31s/it]
|
| 30 |
+
|
| 31 |
+
{'eval_loss': 3.1157305240631104, 'eval_bleu': 3.1978, 'eval_gen_len': 200.0, 'eval_runtime': 0.7589, 'eval_samples_per_second': 6.588, 'eval_steps_per_second': 1.318, 'epoch': 3.0}
|
| 32 |
+
{'train_runtime': 59.66, 'train_samples_per_second': 4.526, 'train_steps_per_second': 0.302, 'train_loss': 3.1893249087863498, 'epoch': 3.0}
|
wandb/run-20250520_132739-ezfnx1ko/files/requirements.txt
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
nvidia-cuda-nvrtc-cu12==12.6.77
|
| 2 |
+
google-cloud-resource-manager==1.14.2
|
| 3 |
+
google-crc32c==1.7.1
|
| 4 |
+
pandas==2.2.3
|
| 5 |
+
tokenizers==0.20.3
|
| 6 |
+
sympy==1.14.0
|
| 7 |
+
typing_extensions==4.13.2
|
| 8 |
+
pymongo==4.13.0
|
| 9 |
+
accelerate==1.7.0
|
| 10 |
+
uvicorn==0.34.2
|
| 11 |
+
nvidia-cuda-runtime-cu12==12.6.77
|
| 12 |
+
nvidia-cublas-cu12==12.6.4.1
|
| 13 |
+
googleapis-common-protos==1.70.0
|
| 14 |
+
annotated-types==0.7.0
|
| 15 |
+
xxhash==3.5.0
|
| 16 |
+
propcache==0.3.1
|
| 17 |
+
datasets==3.6.0
|
| 18 |
+
kfp-pipeline-spec==0.6.0
|
| 19 |
+
lxml==5.4.0
|
| 20 |
+
requests-toolbelt==1.0.0
|
| 21 |
+
smmap==5.0.2
|
| 22 |
+
torch==2.7.0
|
| 23 |
+
google-cloud-bigquery==3.33.0
|
| 24 |
+
colorama==0.4.6
|
| 25 |
+
aiohttp==3.11.18
|
| 26 |
+
triton==3.3.0
|
| 27 |
+
gitdb==4.0.12
|
| 28 |
+
numpy==2.2.6
|
| 29 |
+
dill==0.3.8
|
| 30 |
+
docker-pycreds==0.4.0
|
| 31 |
+
yarl==1.20.0
|
| 32 |
+
packaging==25.0
|
| 33 |
+
sacrebleu==2.5.1
|
| 34 |
+
evaluate==0.4.3
|
| 35 |
+
python-dateutil==2.9.0.post0
|
| 36 |
+
tqdm==4.67.1
|
| 37 |
+
pydantic_core==2.33.2
|
| 38 |
+
anyio==4.9.0
|
| 39 |
+
google-api-core==2.24.2
|
| 40 |
+
certifi==2025.4.26
|
| 41 |
+
GitPython==3.1.44
|
| 42 |
+
kfp-server-api==2.4.0
|
| 43 |
+
requests-oauthlib==2.0.0
|
| 44 |
+
pytz==2025.2
|
| 45 |
+
Jinja2==3.1.6
|
| 46 |
+
nvidia-cudnn-cu12==9.5.1.17
|
| 47 |
+
google-resumable-media==2.7.2
|
| 48 |
+
portalocker==3.1.1
|
| 49 |
+
transformers==4.46.3
|
| 50 |
+
setuptools==66.1.1
|
| 51 |
+
nvidia-nccl-cu12==2.26.2
|
| 52 |
+
google-cloud-core==2.4.3
|
| 53 |
+
google-cloud-pipeline-components==2.20.0
|
| 54 |
+
oauthlib==3.2.2
|
| 55 |
+
sentry-sdk==2.29.1
|
| 56 |
+
setproctitle==1.3.6
|
| 57 |
+
multiprocess==0.70.16
|
| 58 |
+
multidict==6.4.4
|
| 59 |
+
safetensors==0.5.3
|
| 60 |
+
networkx==3.4.2
|
| 61 |
+
google-cloud-aiplatform==1.38.0
|
| 62 |
+
aiosignal==1.3.2
|
| 63 |
+
charset-normalizer==3.4.2
|
| 64 |
+
psutil==7.0.0
|
| 65 |
+
regex==2024.11.6
|
| 66 |
+
google-auth==2.40.1
|
| 67 |
+
PyYAML==6.0.2
|
| 68 |
+
filelock==3.18.0
|
| 69 |
+
h11==0.16.0
|
| 70 |
+
MarkupSafe==3.0.2
|
| 71 |
+
pip==23.0.1
|
| 72 |
+
dnspython==2.7.0
|
| 73 |
+
docstring_parser==0.16
|
| 74 |
+
pyasn1==0.6.1
|
| 75 |
+
sniffio==1.3.1
|
| 76 |
+
kfp==2.13.0
|
| 77 |
+
requests==2.32.3
|
| 78 |
+
pyarrow==20.0.0
|
| 79 |
+
proto-plus==1.26.1
|
| 80 |
+
aiohappyeyeballs==2.6.1
|
| 81 |
+
nvidia-cusparse-cu12==12.5.4.2
|
| 82 |
+
mpmath==1.3.0
|
| 83 |
+
nvidia-nvjitlink-cu12==12.6.85
|
| 84 |
+
attrs==25.3.0
|
| 85 |
+
tabulate==0.9.0
|
| 86 |
+
nvidia-cuda-cupti-cu12==12.6.80
|
| 87 |
+
starlette==0.46.2
|
| 88 |
+
grpcio==1.71.0
|
| 89 |
+
six==1.17.0
|
| 90 |
+
nvidia-cusolver-cu12==11.7.1.2
|
| 91 |
+
nvidia-nvtx-cu12==12.6.77
|
| 92 |
+
wandb==0.19.11
|
| 93 |
+
huggingface-hub==0.31.4
|
| 94 |
+
nvidia-cusparselt-cu12==0.6.3
|
| 95 |
+
urllib3==2.4.0
|
| 96 |
+
nvidia-cufft-cu12==11.3.0.4
|
| 97 |
+
pydantic==2.11.4
|
| 98 |
+
typing-inspection==0.4.0
|
| 99 |
+
grpc-google-iam-v1==0.14.2
|
| 100 |
+
shapely==2.1.1
|
| 101 |
+
idna==3.10
|
| 102 |
+
nvidia-curand-cu12==10.3.7.77
|
| 103 |
+
grpcio-status==1.62.3
|
| 104 |
+
frozenlist==1.6.0
|
| 105 |
+
click==8.2.0
|
| 106 |
+
google-cloud-storage==2.19.0
|
| 107 |
+
rsa==4.9.1
|
| 108 |
+
websocket-client==1.8.0
|
| 109 |
+
nvidia-cufile-cu12==1.11.1.6
|
| 110 |
+
fsspec==2025.3.0
|
| 111 |
+
cachetools==5.5.2
|
| 112 |
+
protobuf==4.25.7
|
| 113 |
+
pyasn1_modules==0.4.2
|
| 114 |
+
kubernetes==30.1.0
|
| 115 |
+
platformdirs==4.3.8
|
| 116 |
+
fastapi==0.115.12
|
| 117 |
+
tzdata==2025.2
|
wandb/run-20250520_132739-ezfnx1ko/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.1.0-35-cloud-amd64-x86_64-with-glibc2.36",
|
| 3 |
+
"python": "CPython 3.11.2",
|
| 4 |
+
"startedAt": "2025-05-20T13:27:39.708945Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--output_dir",
|
| 7 |
+
"models",
|
| 8 |
+
"--wandb_api_key",
|
| 9 |
+
"27b61f72ef87b0a4d86515322e1ed1197a279323"
|
| 10 |
+
],
|
| 11 |
+
"program": "-m scripts.train",
|
| 12 |
+
"email": "koladeodunope@gmail.com",
|
| 13 |
+
"root": "models",
|
| 14 |
+
"host": "nkenneai-gpu",
|
| 15 |
+
"executable": "/home/kolade_esmglobalconsulting_com/Documents/mlops/mlops/bin/python3",
|
| 16 |
+
"cpu_count": 6,
|
| 17 |
+
"cpu_count_logical": 12,
|
| 18 |
+
"gpu": "NVIDIA A100-SXM4-40GB",
|
| 19 |
+
"gpu_count": 1,
|
| 20 |
+
"disk": {
|
| 21 |
+
"/": {
|
| 22 |
+
"total": "528173547520",
|
| 23 |
+
"used": "340865998848"
|
| 24 |
+
}
|
| 25 |
+
},
|
| 26 |
+
"memory": {
|
| 27 |
+
"total": "89632579584"
|
| 28 |
+
},
|
| 29 |
+
"cpu": {
|
| 30 |
+
"count": 6,
|
| 31 |
+
"countLogical": 12
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA A100-SXM4-40GB",
|
| 36 |
+
"memoryTotal": "42949672960",
|
| 37 |
+
"cudaCores": 6912,
|
| 38 |
+
"architecture": "Ampere"
|
| 39 |
+
}
|
| 40 |
+
],
|
| 41 |
+
"cudaVersion": "12.2"
|
| 42 |
+
}
|
wandb/run-20250520_132739-ezfnx1ko/logs/debug-core.log
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2025-05-20T13:27:39.48973059Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpmutquzt2/port-14446.txt","pid":14446,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2025-05-20T13:27:39.49372579Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":14446}
|
| 3 |
+
{"time":"2025-05-20T13:27:39.495746481Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43377,"Zone":""}}
|
| 4 |
+
{"time":"2025-05-20T13:27:39.630524127Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:48972"}
|
| 5 |
+
{"time":"2025-05-20T13:27:39.713706021Z","level":"INFO","msg":"handleInformInit: received","streamId":"ezfnx1ko","id":"127.0.0.1:48972"}
|
| 6 |
+
{"time":"2025-05-20T13:27:39.858589995Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"ezfnx1ko","id":"127.0.0.1:48972"}
|
wandb/run-20250520_132739-ezfnx1ko/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2025-05-20T13:27:39.715003286Z","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"models/wandb/run-20250520_132739-ezfnx1ko/logs/debug-core.log"}
|
| 2 |
+
{"time":"2025-05-20T13:27:39.858492158Z","level":"INFO","msg":"created new stream","id":"ezfnx1ko"}
|
| 3 |
+
{"time":"2025-05-20T13:27:39.858583385Z","level":"INFO","msg":"stream: started","id":"ezfnx1ko"}
|
| 4 |
+
{"time":"2025-05-20T13:27:39.858611644Z","level":"INFO","msg":"writer: Do: started","stream_id":"ezfnx1ko"}
|
| 5 |
+
{"time":"2025-05-20T13:27:39.858799027Z","level":"INFO","msg":"sender: started","stream_id":"ezfnx1ko"}
|
| 6 |
+
{"time":"2025-05-20T13:27:39.858821988Z","level":"INFO","msg":"handler: started","stream_id":"ezfnx1ko"}
|
| 7 |
+
{"time":"2025-05-20T13:27:39.960784281Z","level":"INFO","msg":"Starting system monitor"}
|
wandb/run-20250520_132739-ezfnx1ko/logs/debug.log
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
|
| 2 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_setup.py:_flush():70] Configure stats pid to 14446
|
| 3 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_setup.py:_flush():70] Loading settings from /home/kolade_esmglobalconsulting_com/.config/wandb/settings
|
| 4 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_setup.py:_flush():70] Loading settings from /home/kolade_esmglobalconsulting_com/Documents/mlops/app/wandb/settings
|
| 5 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_setup.py:_flush():70] Loading settings from environment variables
|
| 6 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_init.py:setup_run_log_directory():724] Logging user logs to models/wandb/run-20250520_132739-ezfnx1ko/logs/debug.log
|
| 7 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to models/wandb/run-20250520_132739-ezfnx1ko/logs/debug-internal.log
|
| 8 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_init.py:init():852] calling init triggers
|
| 9 |
+
2025-05-20 13:27:39,702 INFO MainThread:14446 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
|
| 10 |
+
config: {'_wandb': {}}
|
| 11 |
+
2025-05-20 13:27:39,703 INFO MainThread:14446 [wandb_init.py:init():893] starting backend
|
| 12 |
+
2025-05-20 13:27:39,703 INFO MainThread:14446 [wandb_init.py:init():897] sending inform_init request
|
| 13 |
+
2025-05-20 13:27:39,708 INFO MainThread:14446 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
|
| 14 |
+
2025-05-20 13:27:39,708 INFO MainThread:14446 [wandb_init.py:init():907] backend started and connected
|
| 15 |
+
2025-05-20 13:27:39,710 INFO MainThread:14446 [wandb_init.py:init():1005] updated telemetry
|
| 16 |
+
2025-05-20 13:27:39,710 INFO MainThread:14446 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
|
| 17 |
+
2025-05-20 13:27:39,957 INFO MainThread:14446 [wandb_init.py:init():1104] starting run threads in backend
|
| 18 |
+
2025-05-20 13:27:40,144 INFO MainThread:14446 [wandb_run.py:_console_start():2573] atexit reg
|
| 19 |
+
2025-05-20 13:27:40,145 INFO MainThread:14446 [wandb_run.py:_redirect():2421] redirect: wrap_raw
|
| 20 |
+
2025-05-20 13:27:40,145 INFO MainThread:14446 [wandb_run.py:_redirect():2490] Wrapping output streams.
|
| 21 |
+
2025-05-20 13:27:40,145 INFO MainThread:14446 [wandb_run.py:_redirect():2513] Redirects installed.
|
| 22 |
+
2025-05-20 13:27:40,147 INFO MainThread:14446 [wandb_init.py:init():1150] run started, returning control to user process
|
| 23 |
+
2025-05-20 13:27:40,339 INFO MainThread:14446 [wandb_run.py:_config_callback():1436] config_cb None None {'vocab_size': 256206, 'max_position_embeddings': 1024, 'd_model': 1024, 'encoder_ffn_dim': 4096, 'encoder_layers': 12, 'encoder_attention_heads': 16, 'decoder_ffn_dim': 4096, 'decoder_layers': 12, 'decoder_attention_heads': 16, 'dropout': 0.1, 'attention_dropout': 0.1, 'activation_dropout': 0.0, 'activation_function': 'relu', 'init_std': 0.02, 'encoder_layerdrop': 0, 'decoder_layerdrop': 0, 'use_cache': True, 'num_hidden_layers': 12, 'scale_embedding': True, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'float32', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': True, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 200, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['M2M100ForConditionalGeneration'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': 'NllbTokenizer', 'prefix': None, 'bos_token_id': 0, 'pad_token_id': 1, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': 2, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'facebook/nllb-200-distilled-600M', '_attn_implementation_autoset': True, 'transformers_version': '4.46.3', 'model_type': 'm2m_100', 'output_dir': 'models', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'epoch', 'prediction_loss_only': False, 'per_device_train_batch_size': 16, 'per_device_eval_batch_size': 16, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 2e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': 'models/logs', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 500, 'logging_nan_inf_filter': True, 'save_strategy': 'epoch', 'save_steps': 500, 'save_total_limit': 3, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': 'latest_yoruba_mod', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False, 'average_tokens_across_devices': False, 'sortish_sampler': False, 'predict_with_generate': True, 'generation_max_length': None, 'generation_num_beams': None, 'generation_config': None}
|
| 24 |
+
2025-05-20 13:27:40,341 INFO MainThread:14446 [wandb_config.py:__setitem__():154] [no run ID] config set model/num_parameters = 615073792 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7f78cfa89290>>
|
| 25 |
+
2025-05-20 13:27:40,341 INFO MainThread:14446 [wandb_run.py:_config_callback():1436] config_cb model/num_parameters 615073792 None
|
wandb/run-20250520_132739-ezfnx1ko/run-ezfnx1ko.wandb
ADDED
|
Binary file (32.8 kB). View file
|
|
|