Upload batch 1/14: cola/mBERT/attention/seed_42 ... cola/mBERT/gated_multi_branch/seed_42

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/attention/seed_42/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3018324375152588,
+    "eval_matthews_correlation": 0.7469782739691797,
+    "eval_runtime": 1.6368,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 637.222,
+    "eval_steps_per_second": 4.277,
+    "total_flos": 429108409807626.0,
+    "train_loss": 0.24882495366152946,
+    "train_runtime": 26.6404,
+    "train_samples": 8551,
+    "train_samples_per_second": 962.936,
+    "train_steps_per_second": 12.049
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/attention/seed_42/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3018324375152588,
+    "eval_matthews_correlation": 0.7469782739691797,
+    "eval_runtime": 1.6368,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 637.222,
+    "eval_steps_per_second": 4.277
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/attention/seed_42/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 42 --output_dir result_ablation_mbert_paper/cola/mBERT/attention/seed_42 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy attention --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/attention/seed_42/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 429108409807626.0,
+    "train_loss": 0.24882495366152946,
+    "train_runtime": 26.6404,
+    "train_samples": 8551,
+    "train_samples_per_second": 962.936,
+    "train_steps_per_second": 12.049
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/attention/seed_42/trainer_state.json

@@ -0,0 +1,111 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 321,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4672897196261682,
+      "grad_norm": 5.81113862991333,
+      "learning_rate": 1.688473520249221e-05,
+      "loss": 0.4141,
+      "step": 50
+    },
+    {
+      "epoch": 0.9345794392523364,
+      "grad_norm": 3.5820555686950684,
+      "learning_rate": 1.3769470404984425e-05,
+      "loss": 0.3117,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.28091415762901306,
+      "eval_matthews_correlation": 0.7231377957738392,
+      "eval_runtime": 1.6446,
+      "eval_samples_per_second": 634.202,
+      "eval_steps_per_second": 4.256,
+      "step": 107
+    },
+    {
+      "epoch": 1.4018691588785046,
+      "grad_norm": 5.024538516998291,
+      "learning_rate": 1.0654205607476635e-05,
+      "loss": 0.25,
+      "step": 150
+    },
+    {
+      "epoch": 1.8691588785046729,
+      "grad_norm": 4.812474727630615,
+      "learning_rate": 7.538940809968847e-06,
+      "loss": 0.2205,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.2870105504989624,
+      "eval_matthews_correlation": 0.7566293109993708,
+      "eval_runtime": 1.6696,
+      "eval_samples_per_second": 624.699,
+      "eval_steps_per_second": 4.193,
+      "step": 214
+    },
+    {
+      "epoch": 2.336448598130841,
+      "grad_norm": 6.258780002593994,
+      "learning_rate": 4.42367601246106e-06,
+      "loss": 0.185,
+      "step": 250
+    },
+    {
+      "epoch": 2.803738317757009,
+      "grad_norm": 5.915642738342285,
+      "learning_rate": 1.308411214953271e-06,
+      "loss": 0.1552,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.3018324375152588,
+      "eval_matthews_correlation": 0.7469782739691797,
+      "eval_runtime": 1.4542,
+      "eval_samples_per_second": 717.243,
+      "eval_steps_per_second": 4.814,
+      "step": 321
+    },
+    {
+      "epoch": 3.0,
+      "step": 321,
+      "total_flos": 429108409807626.0,
+      "train_loss": 0.24882495366152946,
+      "train_runtime": 26.6404,
+      "train_samples_per_second": 962.936,
+      "train_steps_per_second": 12.049
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 321,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 429108409807626.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/attention/seed_43/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.28296852111816406,
+    "eval_matthews_correlation": 0.7658812546257014,
+    "eval_runtime": 1.422,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 733.484,
+    "eval_steps_per_second": 4.923,
+    "total_flos": 426225629122602.0,
+    "train_loss": 0.26194217346167636,
+    "train_runtime": 30.3278,
+    "train_samples": 8551,
+    "train_samples_per_second": 845.857,
+    "train_steps_per_second": 10.584
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/attention/seed_43/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.28296852111816406,
+    "eval_matthews_correlation": 0.7658812546257014,
+    "eval_runtime": 1.422,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 733.484,
+    "eval_steps_per_second": 4.923
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/attention/seed_43/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 43 --output_dir result_ablation_mbert_paper/cola/mBERT/attention/seed_43 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy attention --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/attention/seed_43/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 426225629122602.0,
+    "train_loss": 0.26194217346167636,
+    "train_runtime": 30.3278,
+    "train_samples": 8551,
+    "train_samples_per_second": 845.857,
+    "train_steps_per_second": 10.584
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/attention/seed_43/trainer_state.json

@@ -0,0 +1,111 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 321,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4672897196261682,
+      "grad_norm": 3.5030007362365723,
+      "learning_rate": 1.688473520249221e-05,
+      "loss": 0.4648,
+      "step": 50
+    },
+    {
+      "epoch": 0.9345794392523364,
+      "grad_norm": 4.771762371063232,
+      "learning_rate": 1.3769470404984425e-05,
+      "loss": 0.3283,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.2850252389907837,
+      "eval_matthews_correlation": 0.7160180415692806,
+      "eval_runtime": 1.6694,
+      "eval_samples_per_second": 624.763,
+      "eval_steps_per_second": 4.193,
+      "step": 107
+    },
+    {
+      "epoch": 1.4018691588785046,
+      "grad_norm": 5.225672245025635,
+      "learning_rate": 1.0654205607476635e-05,
+      "loss": 0.2535,
+      "step": 150
+    },
+    {
+      "epoch": 1.8691588785046729,
+      "grad_norm": 4.631227970123291,
+      "learning_rate": 7.538940809968847e-06,
+      "loss": 0.2213,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.2561209797859192,
+      "eval_matthews_correlation": 0.7698455874888194,
+      "eval_runtime": 1.5514,
+      "eval_samples_per_second": 672.295,
+      "eval_steps_per_second": 4.512,
+      "step": 214
+    },
+    {
+      "epoch": 2.336448598130841,
+      "grad_norm": 7.529651641845703,
+      "learning_rate": 4.42367601246106e-06,
+      "loss": 0.1778,
+      "step": 250
+    },
+    {
+      "epoch": 2.803738317757009,
+      "grad_norm": 3.7077887058258057,
+      "learning_rate": 1.308411214953271e-06,
+      "loss": 0.1662,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.28296852111816406,
+      "eval_matthews_correlation": 0.7658812546257014,
+      "eval_runtime": 1.2873,
+      "eval_samples_per_second": 810.246,
+      "eval_steps_per_second": 5.438,
+      "step": 321
+    },
+    {
+      "epoch": 3.0,
+      "step": 321,
+      "total_flos": 426225629122602.0,
+      "train_loss": 0.26194217346167636,
+      "train_runtime": 30.3278,
+      "train_samples_per_second": 845.857,
+      "train_steps_per_second": 10.584
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 321,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 426225629122602.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/attention/seed_44/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.28675714135169983,
+    "eval_matthews_correlation": 0.7776576303274508,
+    "eval_runtime": 1.3638,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 764.765,
+    "eval_steps_per_second": 5.133,
+    "total_flos": 426289128768780.0,
+    "train_loss": 0.25152042053198886,
+    "train_runtime": 30.2705,
+    "train_samples": 8551,
+    "train_samples_per_second": 847.46,
+    "train_steps_per_second": 10.604
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/attention/seed_44/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.28675714135169983,
+    "eval_matthews_correlation": 0.7776576303274508,
+    "eval_runtime": 1.3638,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 764.765,
+    "eval_steps_per_second": 5.133
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/attention/seed_44/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 44 --output_dir result_ablation_mbert_paper/cola/mBERT/attention/seed_44 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy attention --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/attention/seed_44/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 426289128768780.0,
+    "train_loss": 0.25152042053198886,
+    "train_runtime": 30.2705,
+    "train_samples": 8551,
+    "train_samples_per_second": 847.46,
+    "train_steps_per_second": 10.604
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/attention/seed_44/trainer_state.json

@@ -0,0 +1,111 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 321,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4672897196261682,
+      "grad_norm": 4.501162528991699,
+      "learning_rate": 1.688473520249221e-05,
+      "loss": 0.4328,
+      "step": 50
+    },
+    {
+      "epoch": 0.9345794392523364,
+      "grad_norm": 5.196439266204834,
+      "learning_rate": 1.3769470404984425e-05,
+      "loss": 0.3116,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.25933611392974854,
+      "eval_matthews_correlation": 0.7493314070397258,
+      "eval_runtime": 1.6419,
+      "eval_samples_per_second": 635.241,
+      "eval_steps_per_second": 4.263,
+      "step": 107
+    },
+    {
+      "epoch": 1.4018691588785046,
+      "grad_norm": 5.636826992034912,
+      "learning_rate": 1.0654205607476635e-05,
+      "loss": 0.2497,
+      "step": 150
+    },
+    {
+      "epoch": 1.8691588785046729,
+      "grad_norm": 6.54767370223999,
+      "learning_rate": 7.538940809968847e-06,
+      "loss": 0.218,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.2662619650363922,
+      "eval_matthews_correlation": 0.7823610499496286,
+      "eval_runtime": 1.4422,
+      "eval_samples_per_second": 723.19,
+      "eval_steps_per_second": 4.854,
+      "step": 214
+    },
+    {
+      "epoch": 2.336448598130841,
+      "grad_norm": 9.159810066223145,
+      "learning_rate": 4.42367601246106e-06,
+      "loss": 0.1753,
+      "step": 250
+    },
+    {
+      "epoch": 2.803738317757009,
+      "grad_norm": 10.710851669311523,
+      "learning_rate": 1.308411214953271e-06,
+      "loss": 0.1671,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.28675714135169983,
+      "eval_matthews_correlation": 0.7776576303274508,
+      "eval_runtime": 1.3057,
+      "eval_samples_per_second": 798.8,
+      "eval_steps_per_second": 5.361,
+      "step": 321
+    },
+    {
+      "epoch": 3.0,
+      "step": 321,
+      "total_flos": 426289128768780.0,
+      "train_loss": 0.25152042053198886,
+      "train_runtime": 30.2705,
+      "train_samples_per_second": 847.46,
+      "train_steps_per_second": 10.604
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 321,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 426289128768780.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/cls/seed_42/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3118951916694641,
+    "eval_matthews_correlation": 0.7376962218509274,
+    "eval_runtime": 1.3332,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 782.309,
+    "eval_steps_per_second": 5.25,
+    "total_flos": 429104574692352.0,
+    "train_loss": 0.26396365477659994,
+    "train_runtime": 34.4335,
+    "train_samples": 8551,
+    "train_samples_per_second": 745.0,
+    "train_steps_per_second": 9.322
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/cls/seed_42/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3118951916694641,
+    "eval_matthews_correlation": 0.7376962218509274,
+    "eval_runtime": 1.3332,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 782.309,
+    "eval_steps_per_second": 5.25
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/cls/seed_42/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 42 --output_dir result_ablation_mbert_paper/cola/mBERT/cls/seed_42 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy cls --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/cls/seed_42/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 429104574692352.0,
+    "train_loss": 0.26396365477659994,
+    "train_runtime": 34.4335,
+    "train_samples": 8551,
+    "train_samples_per_second": 745.0,
+    "train_steps_per_second": 9.322
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/cls/seed_42/trainer_state.json

@@ -0,0 +1,111 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 321,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4672897196261682,
+      "grad_norm": 5.808596611022949,
+      "learning_rate": 1.688473520249221e-05,
+      "loss": 0.4422,
+      "step": 50
+    },
+    {
+      "epoch": 0.9345794392523364,
+      "grad_norm": 4.709081649780273,
+      "learning_rate": 1.3769470404984425e-05,
+      "loss": 0.3206,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.31404176354408264,
+      "eval_matthews_correlation": 0.7097825331203318,
+      "eval_runtime": 1.5463,
+      "eval_samples_per_second": 674.531,
+      "eval_steps_per_second": 4.527,
+      "step": 107
+    },
+    {
+      "epoch": 1.4018691588785046,
+      "grad_norm": 4.354961395263672,
+      "learning_rate": 1.0654205607476635e-05,
+      "loss": 0.2704,
+      "step": 150
+    },
+    {
+      "epoch": 1.8691588785046729,
+      "grad_norm": 6.4839277267456055,
+      "learning_rate": 7.538940809968847e-06,
+      "loss": 0.2277,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.28277787566185,
+      "eval_matthews_correlation": 0.73745767326956,
+      "eval_runtime": 1.3714,
+      "eval_samples_per_second": 760.525,
+      "eval_steps_per_second": 5.104,
+      "step": 214
+    },
+    {
+      "epoch": 2.336448598130841,
+      "grad_norm": 7.083432674407959,
+      "learning_rate": 4.42367601246106e-06,
+      "loss": 0.2001,
+      "step": 250
+    },
+    {
+      "epoch": 2.803738317757009,
+      "grad_norm": 5.132438659667969,
+      "learning_rate": 1.308411214953271e-06,
+      "loss": 0.1679,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.3118951916694641,
+      "eval_matthews_correlation": 0.7376962218509274,
+      "eval_runtime": 1.5364,
+      "eval_samples_per_second": 678.855,
+      "eval_steps_per_second": 4.556,
+      "step": 321
+    },
+    {
+      "epoch": 3.0,
+      "step": 321,
+      "total_flos": 429104574692352.0,
+      "train_loss": 0.26396365477659994,
+      "train_runtime": 34.4335,
+      "train_samples_per_second": 745.0,
+      "train_steps_per_second": 9.322
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 321,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 429104574692352.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/cls/seed_43/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.2848812937736511,
+    "eval_matthews_correlation": 0.7682410888691849,
+    "eval_runtime": 1.416,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 736.592,
+    "eval_steps_per_second": 4.944,
+    "total_flos": 426221819771904.0,
+    "train_loss": 0.26999439925790947,
+    "train_runtime": 34.7235,
+    "train_samples": 8551,
+    "train_samples_per_second": 738.78,
+    "train_steps_per_second": 9.244
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/cls/seed_43/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.2848812937736511,
+    "eval_matthews_correlation": 0.7682410888691849,
+    "eval_runtime": 1.416,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 736.592,
+    "eval_steps_per_second": 4.944
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/cls/seed_43/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 43 --output_dir result_ablation_mbert_paper/cola/mBERT/cls/seed_43 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy cls --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/cls/seed_43/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 426221819771904.0,
+    "train_loss": 0.26999439925790947,
+    "train_runtime": 34.7235,
+    "train_samples": 8551,
+    "train_samples_per_second": 738.78,
+    "train_steps_per_second": 9.244
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/cls/seed_43/trainer_state.json

@@ -0,0 +1,111 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 321,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4672897196261682,
+      "grad_norm": 7.262279987335205,
+      "learning_rate": 1.688473520249221e-05,
+      "loss": 0.4513,
+      "step": 50
+    },
+    {
+      "epoch": 0.9345794392523364,
+      "grad_norm": 6.623566150665283,
+      "learning_rate": 1.3769470404984425e-05,
+      "loss": 0.3395,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.2865925431251526,
+      "eval_matthews_correlation": 0.7255313529135801,
+      "eval_runtime": 1.5661,
+      "eval_samples_per_second": 665.996,
+      "eval_steps_per_second": 4.47,
+      "step": 107
+    },
+    {
+      "epoch": 1.4018691588785046,
+      "grad_norm": 6.386998653411865,
+      "learning_rate": 1.0654205607476635e-05,
+      "loss": 0.2696,
+      "step": 150
+    },
+    {
+      "epoch": 1.8691588785046729,
+      "grad_norm": 5.353734970092773,
+      "learning_rate": 7.538940809968847e-06,
+      "loss": 0.2277,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.25154250860214233,
+      "eval_matthews_correlation": 0.7925689872887521,
+      "eval_runtime": 1.5018,
+      "eval_samples_per_second": 694.487,
+      "eval_steps_per_second": 4.661,
+      "step": 214
+    },
+    {
+      "epoch": 2.336448598130841,
+      "grad_norm": 6.868773460388184,
+      "learning_rate": 4.42367601246106e-06,
+      "loss": 0.1907,
+      "step": 250
+    },
+    {
+      "epoch": 2.803738317757009,
+      "grad_norm": 3.198869466781616,
+      "learning_rate": 1.308411214953271e-06,
+      "loss": 0.1804,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.2848812937736511,
+      "eval_matthews_correlation": 0.7682410888691849,
+      "eval_runtime": 1.4122,
+      "eval_samples_per_second": 738.542,
+      "eval_steps_per_second": 4.957,
+      "step": 321
+    },
+    {
+      "epoch": 3.0,
+      "step": 321,
+      "total_flos": 426221819771904.0,
+      "train_loss": 0.26999439925790947,
+      "train_runtime": 34.7235,
+      "train_samples_per_second": 738.78,
+      "train_steps_per_second": 9.244
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 321,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 426221819771904.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/cls/seed_44/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.29706358909606934,
+    "eval_matthews_correlation": 0.7682418720619295,
+    "eval_runtime": 1.3729,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 759.699,
+    "eval_steps_per_second": 5.099,
+    "total_flos": 426285318850560.0,
+    "train_loss": 0.25450210036518417,
+    "train_runtime": 34.6579,
+    "train_samples": 8551,
+    "train_samples_per_second": 740.177,
+    "train_steps_per_second": 9.262
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/cls/seed_44/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.29706358909606934,
+    "eval_matthews_correlation": 0.7682418720619295,
+    "eval_runtime": 1.3729,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 759.699,
+    "eval_steps_per_second": 5.099
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/cls/seed_44/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 44 --output_dir result_ablation_mbert_paper/cola/mBERT/cls/seed_44 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy cls --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/cls/seed_44/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 426285318850560.0,
+    "train_loss": 0.25450210036518417,
+    "train_runtime": 34.6579,
+    "train_samples": 8551,
+    "train_samples_per_second": 740.177,
+    "train_steps_per_second": 9.262
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/cls/seed_44/trainer_state.json

@@ -0,0 +1,111 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 321,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4672897196261682,
+      "grad_norm": 5.091413497924805,
+      "learning_rate": 1.688473520249221e-05,
+      "loss": 0.4267,
+      "step": 50
+    },
+    {
+      "epoch": 0.9345794392523364,
+      "grad_norm": 5.1950764656066895,
+      "learning_rate": 1.3769470404984425e-05,
+      "loss": 0.315,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.259922593832016,
+      "eval_matthews_correlation": 0.7659095653684762,
+      "eval_runtime": 1.5582,
+      "eval_samples_per_second": 669.382,
+      "eval_steps_per_second": 4.492,
+      "step": 107
+    },
+    {
+      "epoch": 1.4018691588785046,
+      "grad_norm": 5.492203712463379,
+      "learning_rate": 1.0654205607476635e-05,
+      "loss": 0.2544,
+      "step": 150
+    },
+    {
+      "epoch": 1.8691588785046729,
+      "grad_norm": 7.6926164627075195,
+      "learning_rate": 7.538940809968847e-06,
+      "loss": 0.2234,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.2667835056781769,
+      "eval_matthews_correlation": 0.7682590021959839,
+      "eval_runtime": 1.4626,
+      "eval_samples_per_second": 713.094,
+      "eval_steps_per_second": 4.786,
+      "step": 214
+    },
+    {
+      "epoch": 2.336448598130841,
+      "grad_norm": 9.207656860351562,
+      "learning_rate": 4.42367601246106e-06,
+      "loss": 0.1838,
+      "step": 250
+    },
+    {
+      "epoch": 2.803738317757009,
+      "grad_norm": 11.206198692321777,
+      "learning_rate": 1.308411214953271e-06,
+      "loss": 0.1684,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.29706358909606934,
+      "eval_matthews_correlation": 0.7682418720619295,
+      "eval_runtime": 1.3802,
+      "eval_samples_per_second": 755.693,
+      "eval_steps_per_second": 5.072,
+      "step": 321
+    },
+    {
+      "epoch": 3.0,
+      "step": 321,
+      "total_flos": 426285318850560.0,
+      "train_loss": 0.25450210036518417,
+      "train_runtime": 34.6579,
+      "train_samples_per_second": 740.177,
+      "train_steps_per_second": 9.262
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 321,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 426285318850560.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_42/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.30131927132606506,
+    "eval_matthews_correlation": 0.7641385489211314,
+    "eval_runtime": 1.5389,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 677.745,
+    "eval_steps_per_second": 4.549,
+    "total_flos": 448737841399356.0,
+    "train_loss": 0.2518068837970959,
+    "train_runtime": 24.6142,
+    "train_samples": 8551,
+    "train_samples_per_second": 1042.203,
+    "train_steps_per_second": 13.041
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_42/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.30131927132606506,
+    "eval_matthews_correlation": 0.7641385489211314,
+    "eval_runtime": 1.5389,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 677.745,
+    "eval_steps_per_second": 4.549
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_42/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 42 --output_dir result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_42 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy gated_multi_branch --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_42/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 448737841399356.0,
+    "train_loss": 0.2518068837970959,
+    "train_runtime": 24.6142,
+    "train_samples": 8551,
+    "train_samples_per_second": 1042.203,
+    "train_steps_per_second": 13.041
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_42/trainer_state.json

@@ -0,0 +1,111 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 321,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4672897196261682,
+      "grad_norm": 4.675717353820801,
+      "learning_rate": 1.688473520249221e-05,
+      "loss": 0.4265,
+      "step": 50
+    },
+    {
+      "epoch": 0.9345794392523364,
+      "grad_norm": 5.895548343658447,
+      "learning_rate": 1.3769470404984425e-05,
+      "loss": 0.3141,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.2743222415447235,
+      "eval_matthews_correlation": 0.7231769633837785,
+      "eval_runtime": 1.4946,
+      "eval_samples_per_second": 697.848,
+      "eval_steps_per_second": 4.684,
+      "step": 107
+    },
+    {
+      "epoch": 1.4018691588785046,
+      "grad_norm": 8.380515098571777,
+      "learning_rate": 1.0654205607476635e-05,
+      "loss": 0.2507,
+      "step": 150
+    },
+    {
+      "epoch": 1.8691588785046729,
+      "grad_norm": 4.941604137420654,
+      "learning_rate": 7.538940809968847e-06,
+      "loss": 0.2166,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.25941094756126404,
+      "eval_matthews_correlation": 0.7706478059846233,
+      "eval_runtime": 1.3362,
+      "eval_samples_per_second": 780.59,
+      "eval_steps_per_second": 5.239,
+      "step": 214
+    },
+    {
+      "epoch": 2.336448598130841,
+      "grad_norm": 4.413182735443115,
+      "learning_rate": 4.42367601246106e-06,
+      "loss": 0.1945,
+      "step": 250
+    },
+    {
+      "epoch": 2.803738317757009,
+      "grad_norm": 7.304749488830566,
+      "learning_rate": 1.308411214953271e-06,
+      "loss": 0.1546,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.30131927132606506,
+      "eval_matthews_correlation": 0.7641385489211314,
+      "eval_runtime": 1.6292,
+      "eval_samples_per_second": 640.191,
+      "eval_steps_per_second": 4.297,
+      "step": 321
+    },
+    {
+      "epoch": 3.0,
+      "step": 321,
+      "total_flos": 448737841399356.0,
+      "train_loss": 0.2518068837970959,
+      "train_runtime": 24.6142,
+      "train_samples_per_second": 1042.203,
+      "train_steps_per_second": 13.041
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 321,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 448737841399356.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}