Training in progress, step 10

README.md

@@ -40,6 +40,18 @@ The following `bitsandbytes` quantization config was used during training:
 - bnb_4bit_use_double_quant: True
 - bnb_4bit_compute_dtype: bfloat16
 
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+
 The following `bitsandbytes` quantization config was used during training:
 - quant_method: bitsandbytes
 - load_in_8bit: False
@@ -56,5 +68,6 @@ The following `bitsandbytes` quantization config was used during training:
 - PEFT 0.5.0
 - PEFT 0.5.0
 - PEFT 0.5.0
+- PEFT 0.5.0
 
 - PEFT 0.5.0

adapter_config.json

@@ -1,6 +1,6 @@
 {
   "auto_mapping": null,
-  "base_model_name_or_path": "tiiuae/falcon-7b",
+  "base_model_name_or_path": "tiiuae/falcon-rw-1b",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:803ab8011b150d790ddb415469edc8be316d3ce8e7ad91e59c32d9798ee958cb
-size 261189453
+oid sha256:f3092124bd2e862488f1cdca59c3894c7cd49f81d0f2467e0291dcc3c57dc5e5
+size 100733709

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9ad3f6d49c7f11f95815510e85c6f8720bb38f23a3c448313f90939db5afad0
-size 100690288
+oid sha256:3ceda947757e8e863957b7c0ac6282dda8be63d6aff7d21ca866b79d3024273c
+size 261131840

config.json

@@ -1,33 +1,33 @@
 {
-  "_name_or_path": "tiiuae/falcon-rw-1b",
-  "alibi": true,
+  "_name_or_path": "tiiuae/falcon-7b",
+  "alibi": false,
   "apply_residual_connection_post_layernorm": false,
   "architectures": [
     "FalconForCausalLM"
   ],
   "attention_dropout": 0.0,
   "auto_map": {
-    "AutoConfig": "tiiuae/falcon-rw-1b--configuration_falcon.FalconConfig",
-    "AutoModel": "tiiuae/falcon-rw-1b--modeling_falcon.FalconModel",
-    "AutoModelForCausalLM": "tiiuae/falcon-rw-1b--modeling_falcon.FalconForCausalLM",
-    "AutoModelForQuestionAnswering": "tiiuae/falcon-rw-1b--modeling_falcon.FalconForQuestionAnswering",
-    "AutoModelForSequenceClassification": "tiiuae/falcon-rw-1b--modeling_falcon.FalconForSequenceClassification",
-    "AutoModelForTokenClassification": "tiiuae/falcon-rw-1b--modeling_falcon.FalconForTokenClassification"
+    "AutoConfig": "tiiuae/falcon-7b--configuration_falcon.FalconConfig",
+    "AutoModel": "tiiuae/falcon-7b--modeling_falcon.FalconModel",
+    "AutoModelForCausalLM": "tiiuae/falcon-7b--modeling_falcon.FalconForCausalLM",
+    "AutoModelForQuestionAnswering": "tiiuae/falcon-7b--modeling_falcon.FalconForQuestionAnswering",
+    "AutoModelForSequenceClassification": "tiiuae/falcon-7b--modeling_falcon.FalconForSequenceClassification",
+    "AutoModelForTokenClassification": "tiiuae/falcon-7b--modeling_falcon.FalconForTokenClassification"
   },
-  "bias": true,
-  "bos_token_id": 1,
-  "eos_token_id": 2,
+  "bias": false,
+  "bos_token_id": 11,
+  "eos_token_id": 11,
   "hidden_dropout": 0.0,
-  "hidden_size": 2048,
+  "hidden_size": 4544,
   "initializer_range": 0.02,
   "layer_norm_epsilon": 1e-05,
   "model_type": "falcon",
-  "multi_query": false,
+  "multi_query": true,
   "new_decoder_architecture": false,
-  "num_attention_heads": 32,
-  "num_hidden_layers": 24,
-  "num_kv_heads": 32,
-  "parallel_attn": false,
+  "num_attention_heads": 71,
+  "num_hidden_layers": 32,
+  "num_kv_heads": 71,
+  "parallel_attn": true,
   "quantization_config": {
     "bnb_4bit_compute_dtype": "bfloat16",
     "bnb_4bit_quant_type": "nf4",
@@ -43,5 +43,5 @@
   "torch_dtype": "bfloat16",
   "transformers_version": "4.34.1",
   "use_cache": false,
-  "vocab_size": 50304
+  "vocab_size": 65024
 }

special_tokens_map.json

@@ -1,17 +1,6 @@
 {
-  "additional_special_tokens": [
-    ">>TITLE<<",
-    ">>ABSTRACT<<",
-    ">>INTRODUCTION<<",
-    ">>SUMMARY<<",
-    ">>COMMENT<<",
-    ">>ANSWER<<",
-    ">>QUESTION<<",
-    ">>DOMAIN<<",
-    ">>PREFIX<<",
-    ">>SUFFIX<<",
-    ">>MIDDLE<<"
-  ],
+  "bos_token": "<|endoftext|>",
   "eos_token": "<|endoftext|>",
-  "pad_token": "<|endoftext|>"
+  "pad_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
 }

tokenizer_config.json

@@ -1,95 +1,7 @@
 {
   "add_prefix_space": false,
   "added_tokens_decoder": {
-    "0": {
-      "content": ">>TITLE<<",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": ">>ABSTRACT<<",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": ">>INTRODUCTION<<",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "3": {
-      "content": ">>SUMMARY<<",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "4": {
-      "content": ">>COMMENT<<",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "5": {
-      "content": ">>ANSWER<<",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "6": {
-      "content": ">>QUESTION<<",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "7": {
-      "content": ">>DOMAIN<<",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "8": {
-      "content": ">>PREFIX<<",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "9": {
-      "content": ">>SUFFIX<<",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "10": {
-      "content": ">>MIDDLE<<",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "11": {
+    "50256": {
       "content": "<|endoftext|>",
       "lstrip": false,
       "normalized": false,
@@ -98,26 +10,11 @@
       "special": true
     }
   },
-  "additional_special_tokens": [
-    ">>TITLE<<",
-    ">>ABSTRACT<<",
-    ">>INTRODUCTION<<",
-    ">>SUMMARY<<",
-    ">>COMMENT<<",
-    ">>ANSWER<<",
-    ">>QUESTION<<",
-    ">>DOMAIN<<",
-    ">>PREFIX<<",
-    ">>SUFFIX<<",
-    ">>MIDDLE<<"
-  ],
+  "bos_token": "<|endoftext|>",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|endoftext|>",
-  "model_input_names": [
-    "input_ids",
-    "attention_mask"
-  ],
-  "model_max_length": 2048,
+  "model_max_length": 1024,
   "pad_token": "<|endoftext|>",
-  "tokenizer_class": "PreTrainedTokenizerFast"
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93e264e6d806d2410820009c1ca281477c923bfd5f3b234f0fd19dc5a10ed022
+oid sha256:46dfa040a93c7f61a97e86a998ade17bf1b307d0aa4af935c733a7c7ad2cd6b3
 size 4091