Upload 8 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+SAC_5x/output/500xCompressor_15x/output/info.json filter=lfs diff=lfs merge=lfs -text

SAC_5x/output/500xCompressor_15x/config.json

@@ -0,0 +1,58 @@
+{
+    "pretrain_training_config": {
+        "model_id": "/models/meta-llama/Llama-3.2-1B",
+        "chunk_size": 510,
+        "total_batch_size": 16,
+        "batch_size_per_device": 1,
+        "device_count": 8,
+        "gradient_accumulation_steps": 2,
+        "learning_rate": 0.0001,
+        "max_grad_norm": 2.0,
+        "log_step": 100,
+        "valid_step": 100000000,
+        "save_step": 1000
+    },
+    "pretrain_task_config": {
+        "is_pretrain":true,
+        "is_sft":false,
+        "chunk_size": 510,
+        "mem_size": 34,
+        "compress_ratio": 15,
+        "task_type": "Compress",
+        "use_pe": false,
+        "use_ae_loss": true,
+        "use_lm_loss": true
+    },
+    "sft_training_config": {
+        "model_id": "/models/meta-llama/Llama-3.2-1B",
+        "chunk_size": 510,
+        "total_batch_size": 16,
+        "batch_size_per_device": 1,
+        "device_count": 8,
+        "gradient_accumulation_steps": 2,
+        "learning_rate": 0.00005,
+        "max_grad_norm": 2.0,
+        "log_step": 100,
+        "valid_step": 100000000,
+        "save_step": 100000000
+    },
+    "sft_task_config": {
+        "is_pretrain":false,
+        "is_sft":true,
+        "chunk_size": 510,
+        "mem_size": 34,
+        "compress_ratio": 15,
+        "task_type": "Compress",
+        "use_pe": false,
+        "use_ae_loss": false,
+        "use_lm_loss": true
+    },
+    "data_config": {
+        "dataset_repo": "/dataset/DKYoon/SlimPajama-6B",
+        "samples_num": 320000,
+        "min_len": 510,
+        "max_len": 2040,
+        "model_id": "/models/meta-llama/Llama-3.2-1B",
+        "instruction_dataset_repo": "/dataset/bigData/mrqa-workshop_mrqa"
+    }
+}

SAC_5x/output/500xCompressor_15x/output/adapter.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2db581332363b21a096431d98d9a4f84c56dd81791fb66f917aa1c955685dfaf
+size 27432208

SAC_5x/output/500xCompressor_15x/output/config.json

@@ -0,0 +1,62 @@
+{
+    "pretrain_training_config": {
+        "model_id": "/models/meta-llama/Llama-3.2-1B",
+        "chunk_size": 510,
+        "total_batch_size": 16,
+        "batch_size_per_device": 1,
+        "device_count": 8,
+        "gradient_accumulation_steps": 2,
+        "learning_rate": 0.0001,
+        "max_grad_norm": 2.0,
+        "log_step": 100,
+        "valid_step": 100000000,
+        "save_step": 1000
+    },
+    "pretrain_task_config": {
+        "is_pretrain": true,
+        "is_sft": false,
+        "chunk_size": 510,
+        "mem_size": 34,
+        "compress_ratio": 15,
+        "task_type": "Compress",
+        "use_pe": false,
+        "use_ae_loss": true,
+        "use_lm_loss": true
+    },
+    "sft_training_config": {
+        "model_id": "/models/meta-llama/Llama-3.2-1B",
+        "chunk_size": 510,
+        "total_batch_size": 16,
+        "batch_size_per_device": 1,
+        "device_count": 8,
+        "gradient_accumulation_steps": 2,
+        "learning_rate": 5e-05,
+        "max_grad_norm": 2.0,
+        "log_step": 100,
+        "valid_step": 100000000,
+        "save_step": 100000000
+    },
+    "sft_task_config": {
+        "is_pretrain": false,
+        "is_sft": true,
+        "chunk_size": 510,
+        "mem_size": 34,
+        "compress_ratio": 15,
+        "task_type": "Compress",
+        "use_pe": false,
+        "use_ae_loss": false,
+        "use_lm_loss": true
+    },
+    "data_config": {
+        "dataset_repo": "/dataset/DKYoon/SlimPajama-6B",
+        "samples_num": 320000,
+        "min_len": 510,
+        "max_len": 2040,
+        "model_id": "/models/meta-llama/Llama-3.2-1B",
+        "instruction_dataset_repo": "/dataset/bigData/mrqa-workshop_mrqa"
+    },
+    "Total_parameters": 2485334016,
+    "Trainable_parameters": 13705216,
+    "Embedding_parameters": 525336576,
+    "non-Embedding_parameters": 1959997440
+}

SAC_5x/output/500xCompressor_15x/output/iid_subset_eval_results.json

@@ -0,0 +1,26 @@
+{
+    "SQuAD_rouge-f1": 0.40682045116649246,
+    "SQuAD_bleu4": 0.07379134789527392,
+    "SQuAD_exact_match": 0.24973826972494528,
+    "NewsQA_rouge-f1": 0.3200969874396085,
+    "NewsQA_bleu4": 0.06871809731881225,
+    "NewsQA_exact_match": 0.16761633428300096,
+    "TriviaQA-web_rouge-f1": 0.5383565231507547,
+    "TriviaQA-web_bleu4": 0.08955738103663755,
+    "TriviaQA-web_exact_match": 0.4485549132947977,
+    "SearchQA_rouge-f1": 0.6565159386403487,
+    "SearchQA_bleu4": 0.11184616809088285,
+    "SearchQA_exact_match": 0.5369846878680801,
+    "HotpotQA_rouge-f1": 0.5300991272498248,
+    "HotpotQA_bleu4": 0.14151205002184536,
+    "HotpotQA_exact_match": 0.36298932384341637,
+    "NaturalQuestionsShort_rouge-f1": 0.5093019858055343,
+    "NaturalQuestionsShort_bleu4": 0.13109696692067063,
+    "NaturalQuestionsShort_exact_match": 0.3325802430663758,
+    "total_rouge-f1": 0.5260468284990771,
+    "iid_test_samples_num_rouge-f1": 58221,
+    "total_bleu4": 0.10612908981928397,
+    "iid_test_samples_num_bleu4": 58221,
+    "total_exact_match": 0.3838992803284038,
+    "iid_test_samples_num_exact_match": 58221
+}

SAC_5x/output/500xCompressor_15x/output/info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd8ea2a94119fdd31af0227c6af52bf7bdbc6ca2a70ba925c36810bc47060809
+size 10794584

SAC_5x/output/500xCompressor_15x/output/instruction_adapter.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:579dfa8248f5420afca53d162313fb5e3820ec4615d5bfbefcec061c9a9fdcb2
+size 27433624

SAC_5x/output/500xCompressor_15x/output/ood_subset_eval_results.json

@@ -0,0 +1,24 @@
+{
+    "BioASQ_f1": 0.3630254489882323,
+    "BioASQ_bleu4": 0.16459602607963886,
+    "BioASQ_em": 0.25930851063829785,
+    "DROP_f1": 0.3346040942750249,
+    "DROP_bleu4": 0.07076208622312768,
+    "DROP_em": 0.23552894211576847,
+    "DuoRC.ParaphraseRC_f1": 0.20534864557116408,
+    "DuoRC.ParaphraseRC_bleu4": 0.011180489831198855,
+    "DuoRC.ParaphraseRC_em": 0.12724850099933377,
+    "RACE_f1": 0.18486147393418376,
+    "RACE_bleu4": 0.011258001292350515,
+    "RACE_em": 0.03412462908011869,
+    "RelationExtraction_f1": 0.5436664300943114,
+    "RelationExtraction_bleu4": 0.14282737049387742,
+    "RelationExtraction_em": 0.41112618724559025,
+    "TextbookQA_f1": 0.41094585386193655,
+    "TextbookQA_bleu4": 0.029594912302991508,
+    "TextbookQA_em": 0.25815036593479707,
+    "total_f1": 0.38431490020555814,
+    "total_bleu4": 0.08759616820639035,
+    "total_em": 0.26554552060624936,
+    "ood_test_samples_num": 9633
+}