diff --git a/.gitattributes b/.gitattributes index eafc1323824bfa42cb82e67a35b28478de33b094..70025557826f16600f57dcd128b56d2783224bbd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -43,3 +43,6 @@ llmstream_results/llama31_8b_inst_streamline_slimorca_prune8_start21/tokenizer.j llmstream_results/qwen_8b_inst_streamline_slimorca_prune14_start12/tokenizer.json filter=lfs diff=lfs merge=lfs -text llmstream_results/qwen_8b_inst_streamline_slimorca_prune5_start26/tokenizer.json filter=lfs diff=lfs merge=lfs -text llmstream_results/qwen_8b_inst_streamline_slimorca_prune9_start24/tokenizer.json filter=lfs diff=lfs merge=lfs -text +llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/tokenizer.json filter=lfs diff=lfs merge=lfs -text +llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/tokenizer.json filter=lfs diff=lfs merge=lfs -text +llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/config.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/config.json new file mode 100644 index 0000000000000000000000000000000000000000..000e666b52da6502de626edcddc475e18c0369a8 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/config.json @@ -0,0 +1,35 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 20, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 8.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "transformers_version": "4.57.6", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/generation_config.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bab13024abb71f342624ac72ec1a8140082ce13d --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/generation_config.json @@ -0,0 +1,6 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": 128001, + "transformers_version": "4.57.6" +} diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_10/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_10/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef250c42b81e77c11522916e0cae1a5380762517 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_10/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbe3fa0b1b792c3983cbff3082d1680ac7934e2bb5511ac2e064878369702e05 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_12/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_12/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..78653f4150995a4d0ae24b56ce3f42459641a90d --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_12/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd9e863f28ecdbdda384e5e23726533364104349a5a1706192466f2df5554eda +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_14/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_14/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..f8b4560d37d12183fbc8753c9e5089819817b764 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_14/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99daf4e54f9380b6cbc9117650b1c4d4b593753cc0c62efb81116ed76b222fe9 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_16/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_16/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..cdfaa6b3ef2cba3aa832ca5187e3eb967e72e270 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_16/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ac00fc52880d9e08721724f980bd51ecd21e7f13c8cad112d796570a0da5dee +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_18/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_18/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..7193b33ca02a5b1041673bfc4921798bf13a2454 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_18/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7f3f82cc934ea8ed95fdecd59a0c7ee5c618fed2583fe8709916000b1c8b9cc +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_2/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_2/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..7673cc8f52884bbfef20ec61bc12f19716e91e36 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_2/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85526028c6e0ca34b31969d51c21e8d3a7ef791c0b79ee7026f64fda7bfee974 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_20/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_20/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..5dd9744b8cc4f625def6f6f7b1161d65599d7bbb --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_20/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0f7a935e493181717ca4b2306ac53aedc3969de987e36e823c175c6d208a197 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_4/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_4/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..36300480e14f09ea9b7c0e3016447a0848896594 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_4/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d097f22b5a49579a3cf309e17eddb4f175154eeba1968e857db7093181d6be04 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_6/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_6/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..50c09b37cd09cf31ee48414de666d32ef8b5eb9b --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_6/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2246572905c5e931285a9276cdc53fd17ffa581ebebfabd1e3fe0bdc520f6fe4 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_8/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_8/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..4fc457dd0bceee6d08f09789975f7c70fe23fba4 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_epoch_8/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a71c263e4b1e34f33e6f629121c8285ba36ddb40a4fbce1d55084c6ab3c5cc9 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_final/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_final/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..5dd9744b8cc4f625def6f6f7b1161d65599d7bbb --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/lightweight_checkpoint_final/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0f7a935e493181717ca4b2306ac53aedc3969de987e36e823c175c6d208a197 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/model-00001-of-00003.safetensors b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/model-00001-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a70cf544ea82b645a9084d345203e3b855c474af --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8b9704ab09cdeb097aa4a0a24bca96f906eec36bad63ab495bc21475058601b +size 4976698672 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/model-00002-of-00003.safetensors b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/model-00002-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3d9b729c666e0ba7425d910b0ba46d4571494e2d --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9477ef10fb23a13f7acbf016813c11cba8b8fcdd5dc9a798e35b84ff7d313fa +size 4798483816 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/model-00003-of-00003.safetensors b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/model-00003-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..65f02a8fca91b04a36c6ef5d8387979802fafed6 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6feef628d893df7809d975f4c3a109a655e5e4126ac4ee28e8e0464504c86b12 +size 1050673280 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/model.safetensors.index.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..8dd4db268dd812c7f0ce875d51db11797cc6c3fe --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/model.safetensors.index.json @@ -0,0 +1,191 @@ +{ + "metadata": { + "total_parameters": 5412917248, + "total_size": 10825834496 + }, + "weight_map": { + "lm_head.weight": "model-00003-of-00003.safetensors", + "model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.norm.weight": "model-00002-of-00003.safetensors" + } +} diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/run_args.txt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/run_args.txt new file mode 100644 index 0000000000000000000000000000000000000000..ed813250e5a12b46737220b0a7adcad013481c14 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/run_args.txt @@ -0,0 +1,9 @@ +git_commit=unknown +dataset=slimpajama +dataset_split=train +collect_batch_size=8 +train_batch_size=32 +gradient_accumulation_step=8 +target_effective_batch=256 +command: +python /workspace/here/abprune_update/compare_model/LLM-Streamline/mseloss_entry.py --model_name meta-llama/Llama-3.1-8B --output_dir /workspace/here/abprune_update/results/llama31_8b_base_streamline_slimpajama_prune12_start18 --dataset slimpajama --dataset_split train --layer_intervals 12 --best_layer 18 --cosine_num_data 300 --train_num_data 30000 --epoches 20 --batch_size 8 --collect_batch_size 8 --train_batch_size 32 --dtype bfloat16 --gradient_accumulation_step 8 --lr 1e-5 --min_lr 5e-5 --wd 1e-3 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/special_tokens_map.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..04829afa78a2d2df203ac846968db37269b01f7f --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/special_tokens_map.json @@ -0,0 +1,17 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/tokenizer.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/tokenizer_config.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3026161bf4fc56964bad68d35d36b5b815a3f716 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/tokenizer_config.json @@ -0,0 +1,2063 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/train_loss.txt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/train_loss.txt new file mode 100644 index 0000000000000000000000000000000000000000..1585e60784e478740dae9e88e3a34b4a539e8cba --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune12_start18/train_loss.txt @@ -0,0 +1,188 @@ +# Train loss logged every 100 global steps +epoch=0 global_step=100 optimizer_step=12 loss=0.6015625000 +epoch=0 global_step=200 optimizer_step=24 loss=0.5468750000 +epoch=0 global_step=300 optimizer_step=37 loss=0.5585937500 +epoch=0 global_step=400 optimizer_step=49 loss=0.5703125000 +epoch=0 global_step=500 optimizer_step=62 loss=0.5781250000 +epoch=0 global_step=600 optimizer_step=74 loss=0.5664062500 +epoch=0 global_step=700 optimizer_step=87 loss=0.5507812500 +epoch=0 global_step=800 optimizer_step=99 loss=0.5546875000 +epoch=0 global_step=900 optimizer_step=112 loss=0.4902343750 +epoch=1 global_step=1000 optimizer_step=124 loss=0.4453125000 +epoch=1 global_step=1100 optimizer_step=137 loss=0.4453125000 +epoch=1 global_step=1200 optimizer_step=149 loss=0.4277343750 +epoch=1 global_step=1300 optimizer_step=162 loss=0.4375000000 +epoch=1 global_step=1400 optimizer_step=174 loss=0.4296875000 +epoch=1 global_step=1500 optimizer_step=187 loss=0.4355468750 +epoch=1 global_step=1600 optimizer_step=199 loss=0.4238281250 +epoch=1 global_step=1700 optimizer_step=212 loss=0.4218750000 +epoch=1 global_step=1800 optimizer_step=224 loss=0.4160156250 +epoch=2 global_step=1900 optimizer_step=236 loss=0.4179687500 +epoch=2 global_step=2000 optimizer_step=249 loss=0.4238281250 +epoch=2 global_step=2100 optimizer_step=261 loss=0.4160156250 +epoch=2 global_step=2200 optimizer_step=274 loss=0.4121093750 +epoch=2 global_step=2300 optimizer_step=286 loss=0.4335937500 +epoch=2 global_step=2400 optimizer_step=299 loss=0.4238281250 +epoch=2 global_step=2500 optimizer_step=311 loss=0.3964843750 +epoch=2 global_step=2600 optimizer_step=324 loss=0.4199218750 +epoch=2 global_step=2700 optimizer_step=336 loss=0.4218750000 +epoch=2 global_step=2800 optimizer_step=349 loss=0.4101562500 +epoch=3 global_step=2900 optimizer_step=361 loss=0.4082031250 +epoch=3 global_step=3000 optimizer_step=374 loss=0.4042968750 +epoch=3 global_step=3100 optimizer_step=386 loss=0.4042968750 +epoch=3 global_step=3200 optimizer_step=399 loss=0.3945312500 +epoch=3 global_step=3300 optimizer_step=411 loss=0.4042968750 +epoch=3 global_step=3400 optimizer_step=424 loss=0.4003906250 +epoch=3 global_step=3500 optimizer_step=436 loss=0.4140625000 +epoch=3 global_step=3600 optimizer_step=449 loss=0.3984375000 +epoch=3 global_step=3700 optimizer_step=461 loss=0.3886718750 +epoch=4 global_step=3800 optimizer_step=473 loss=0.3945312500 +epoch=4 global_step=3900 optimizer_step=486 loss=0.3925781250 +epoch=4 global_step=4000 optimizer_step=498 loss=0.3906250000 +epoch=4 global_step=4100 optimizer_step=511 loss=0.3906250000 +epoch=4 global_step=4200 optimizer_step=523 loss=0.3789062500 +epoch=4 global_step=4300 optimizer_step=536 loss=0.3867187500 +epoch=4 global_step=4400 optimizer_step=548 loss=0.3886718750 +epoch=4 global_step=4500 optimizer_step=561 loss=0.3925781250 +epoch=4 global_step=4600 optimizer_step=573 loss=0.3867187500 +epoch=5 global_step=4700 optimizer_step=586 loss=0.3769531250 +epoch=5 global_step=4800 optimizer_step=598 loss=0.3925781250 +epoch=5 global_step=4900 optimizer_step=611 loss=0.3886718750 +epoch=5 global_step=5000 optimizer_step=623 loss=0.3808593750 +epoch=5 global_step=5100 optimizer_step=636 loss=0.3945312500 +epoch=5 global_step=5200 optimizer_step=648 loss=0.3828125000 +epoch=5 global_step=5300 optimizer_step=661 loss=0.3886718750 +epoch=5 global_step=5400 optimizer_step=673 loss=0.3886718750 +epoch=5 global_step=5500 optimizer_step=686 loss=0.3769531250 +epoch=5 global_step=5600 optimizer_step=698 loss=0.3808593750 +epoch=6 global_step=5700 optimizer_step=710 loss=0.3671875000 +epoch=6 global_step=5800 optimizer_step=723 loss=0.3691406250 +epoch=6 global_step=5900 optimizer_step=735 loss=0.3808593750 +epoch=6 global_step=6000 optimizer_step=748 loss=0.3730468750 +epoch=6 global_step=6100 optimizer_step=760 loss=0.3808593750 +epoch=6 global_step=6200 optimizer_step=773 loss=0.3847656250 +epoch=6 global_step=6300 optimizer_step=785 loss=0.3671875000 +epoch=6 global_step=6400 optimizer_step=798 loss=0.3671875000 +epoch=6 global_step=6500 optimizer_step=810 loss=0.3867187500 +epoch=7 global_step=6600 optimizer_step=823 loss=0.3593750000 +epoch=7 global_step=6700 optimizer_step=835 loss=0.3847656250 +epoch=7 global_step=6800 optimizer_step=848 loss=0.3789062500 +epoch=7 global_step=6900 optimizer_step=860 loss=0.3769531250 +epoch=7 global_step=7000 optimizer_step=873 loss=0.3691406250 +epoch=7 global_step=7100 optimizer_step=885 loss=0.3691406250 +epoch=7 global_step=7200 optimizer_step=898 loss=0.3632812500 +epoch=7 global_step=7300 optimizer_step=910 loss=0.3750000000 +epoch=7 global_step=7400 optimizer_step=923 loss=0.3652343750 +epoch=7 global_step=7500 optimizer_step=935 loss=0.3886718750 +epoch=8 global_step=7600 optimizer_step=947 loss=0.3886718750 +epoch=8 global_step=7700 optimizer_step=960 loss=0.3671875000 +epoch=8 global_step=7800 optimizer_step=972 loss=0.3632812500 +epoch=8 global_step=7900 optimizer_step=985 loss=0.3808593750 +epoch=8 global_step=8000 optimizer_step=997 loss=0.3710937500 +epoch=8 global_step=8100 optimizer_step=1010 loss=0.3632812500 +epoch=8 global_step=8200 optimizer_step=1022 loss=0.3769531250 +epoch=8 global_step=8300 optimizer_step=1035 loss=0.3691406250 +epoch=8 global_step=8400 optimizer_step=1047 loss=0.3632812500 +epoch=9 global_step=8500 optimizer_step=1060 loss=0.3593750000 +epoch=9 global_step=8600 optimizer_step=1072 loss=0.3750000000 +epoch=9 global_step=8700 optimizer_step=1085 loss=0.3710937500 +epoch=9 global_step=8800 optimizer_step=1097 loss=0.3789062500 +epoch=9 global_step=8900 optimizer_step=1110 loss=0.3789062500 +epoch=9 global_step=9000 optimizer_step=1122 loss=0.3652343750 +epoch=9 global_step=9100 optimizer_step=1135 loss=0.3730468750 +epoch=9 global_step=9200 optimizer_step=1147 loss=0.3652343750 +epoch=9 global_step=9300 optimizer_step=1160 loss=0.3671875000 +epoch=10 global_step=9400 optimizer_step=1172 loss=0.3652343750 +epoch=10 global_step=9500 optimizer_step=1184 loss=0.3652343750 +epoch=10 global_step=9600 optimizer_step=1197 loss=0.3691406250 +epoch=10 global_step=9700 optimizer_step=1209 loss=0.3671875000 +epoch=10 global_step=9800 optimizer_step=1222 loss=0.3652343750 +epoch=10 global_step=9900 optimizer_step=1234 loss=0.3554687500 +epoch=10 global_step=10000 optimizer_step=1247 loss=0.3574218750 +epoch=10 global_step=10100 optimizer_step=1259 loss=0.3730468750 +epoch=10 global_step=10200 optimizer_step=1272 loss=0.3750000000 +epoch=10 global_step=10300 optimizer_step=1284 loss=0.3671875000 +epoch=11 global_step=10400 optimizer_step=1297 loss=0.3457031250 +epoch=11 global_step=10500 optimizer_step=1309 loss=0.3554687500 +epoch=11 global_step=10600 optimizer_step=1322 loss=0.3613281250 +epoch=11 global_step=10700 optimizer_step=1334 loss=0.3730468750 +epoch=11 global_step=10800 optimizer_step=1347 loss=0.3652343750 +epoch=11 global_step=10900 optimizer_step=1359 loss=0.3671875000 +epoch=11 global_step=11000 optimizer_step=1372 loss=0.3613281250 +epoch=11 global_step=11100 optimizer_step=1384 loss=0.3671875000 +epoch=11 global_step=11200 optimizer_step=1397 loss=0.3847656250 +epoch=12 global_step=11300 optimizer_step=1409 loss=0.3593750000 +epoch=12 global_step=11400 optimizer_step=1421 loss=0.3828125000 +epoch=12 global_step=11500 optimizer_step=1434 loss=0.3613281250 +epoch=12 global_step=11600 optimizer_step=1446 loss=0.3632812500 +epoch=12 global_step=11700 optimizer_step=1459 loss=0.3730468750 +epoch=12 global_step=11800 optimizer_step=1471 loss=0.3652343750 +epoch=12 global_step=11900 optimizer_step=1484 loss=0.3574218750 +epoch=12 global_step=12000 optimizer_step=1496 loss=0.3750000000 +epoch=12 global_step=12100 optimizer_step=1509 loss=0.3613281250 +epoch=13 global_step=12200 optimizer_step=1521 loss=0.3515625000 +epoch=13 global_step=12300 optimizer_step=1534 loss=0.3574218750 +epoch=13 global_step=12400 optimizer_step=1546 loss=0.3574218750 +epoch=13 global_step=12500 optimizer_step=1559 loss=0.3691406250 +epoch=13 global_step=12600 optimizer_step=1571 loss=0.3671875000 +epoch=13 global_step=12700 optimizer_step=1584 loss=0.3691406250 +epoch=13 global_step=12800 optimizer_step=1596 loss=0.3613281250 +epoch=13 global_step=12900 optimizer_step=1609 loss=0.3613281250 +epoch=13 global_step=13000 optimizer_step=1621 loss=0.3574218750 +epoch=13 global_step=13100 optimizer_step=1634 loss=0.3652343750 +epoch=14 global_step=13200 optimizer_step=1646 loss=0.3632812500 +epoch=14 global_step=13300 optimizer_step=1658 loss=0.3593750000 +epoch=14 global_step=13400 optimizer_step=1671 loss=0.3652343750 +epoch=14 global_step=13500 optimizer_step=1683 loss=0.3554687500 +epoch=14 global_step=13600 optimizer_step=1696 loss=0.3593750000 +epoch=14 global_step=13700 optimizer_step=1708 loss=0.3496093750 +epoch=14 global_step=13800 optimizer_step=1721 loss=0.3593750000 +epoch=14 global_step=13900 optimizer_step=1733 loss=0.3535156250 +epoch=14 global_step=14000 optimizer_step=1746 loss=0.3652343750 +epoch=15 global_step=14100 optimizer_step=1758 loss=0.3593750000 +epoch=15 global_step=14200 optimizer_step=1771 loss=0.3535156250 +epoch=15 global_step=14300 optimizer_step=1783 loss=0.3593750000 +epoch=15 global_step=14400 optimizer_step=1796 loss=0.3535156250 +epoch=15 global_step=14500 optimizer_step=1808 loss=0.3554687500 +epoch=15 global_step=14600 optimizer_step=1821 loss=0.3593750000 +epoch=15 global_step=14700 optimizer_step=1833 loss=0.3554687500 +epoch=15 global_step=14800 optimizer_step=1846 loss=0.3613281250 +epoch=15 global_step=14900 optimizer_step=1858 loss=0.3750000000 +epoch=15 global_step=15000 optimizer_step=1871 loss=0.3476562500 +epoch=16 global_step=15100 optimizer_step=1883 loss=0.3515625000 +epoch=16 global_step=15200 optimizer_step=1895 loss=0.3554687500 +epoch=16 global_step=15300 optimizer_step=1908 loss=0.3632812500 +epoch=16 global_step=15400 optimizer_step=1920 loss=0.3496093750 +epoch=16 global_step=15500 optimizer_step=1933 loss=0.3515625000 +epoch=16 global_step=15600 optimizer_step=1945 loss=0.3632812500 +epoch=16 global_step=15700 optimizer_step=1958 loss=0.3496093750 +epoch=16 global_step=15800 optimizer_step=1970 loss=0.3632812500 +epoch=16 global_step=15900 optimizer_step=1983 loss=0.3437500000 +epoch=17 global_step=16000 optimizer_step=1995 loss=0.3496093750 +epoch=17 global_step=16100 optimizer_step=2008 loss=0.3476562500 +epoch=17 global_step=16200 optimizer_step=2020 loss=0.3496093750 +epoch=17 global_step=16300 optimizer_step=2033 loss=0.3496093750 +epoch=17 global_step=16400 optimizer_step=2045 loss=0.3554687500 +epoch=17 global_step=16500 optimizer_step=2058 loss=0.3652343750 +epoch=17 global_step=16600 optimizer_step=2070 loss=0.3535156250 +epoch=17 global_step=16700 optimizer_step=2083 loss=0.3652343750 +epoch=17 global_step=16800 optimizer_step=2095 loss=0.3593750000 +epoch=18 global_step=16900 optimizer_step=2107 loss=0.3593750000 +epoch=18 global_step=17000 optimizer_step=2120 loss=0.3632812500 +epoch=18 global_step=17100 optimizer_step=2132 loss=0.3535156250 +epoch=18 global_step=17200 optimizer_step=2145 loss=0.3593750000 +epoch=18 global_step=17300 optimizer_step=2157 loss=0.3535156250 +epoch=18 global_step=17400 optimizer_step=2170 loss=0.3554687500 +epoch=18 global_step=17500 optimizer_step=2182 loss=0.3515625000 +epoch=18 global_step=17600 optimizer_step=2195 loss=0.3574218750 +epoch=18 global_step=17700 optimizer_step=2207 loss=0.3437500000 +epoch=18 global_step=17800 optimizer_step=2220 loss=0.3437500000 +epoch=19 global_step=17900 optimizer_step=2232 loss=0.3535156250 +epoch=19 global_step=18000 optimizer_step=2245 loss=0.3535156250 +epoch=19 global_step=18100 optimizer_step=2257 loss=0.3437500000 +epoch=19 global_step=18200 optimizer_step=2270 loss=0.3398437500 +epoch=19 global_step=18300 optimizer_step=2282 loss=0.3437500000 +epoch=19 global_step=18400 optimizer_step=2295 loss=0.3496093750 +epoch=19 global_step=18500 optimizer_step=2307 loss=0.3496093750 +epoch=19 global_step=18600 optimizer_step=2320 loss=0.3593750000 +epoch=19 global_step=18700 optimizer_step=2332 loss=0.3554687500 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/config.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/config.json new file mode 100644 index 0000000000000000000000000000000000000000..732afbcb88c083ad704168e7956e3a7eba214686 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/config.json @@ -0,0 +1,35 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 8.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "transformers_version": "4.57.6", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/generation_config.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bab13024abb71f342624ac72ec1a8140082ce13d --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/generation_config.json @@ -0,0 +1,6 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": 128001, + "transformers_version": "4.57.6" +} diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_10/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_10/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..eed5776c05f9590aeaa8b7b0f4a72b40acf4161a --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_10/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d66fcc3113d7b51c41c68ba801774d66ec64391786611f01159bc2ee548ab69 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_12/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_12/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..428ec41b7e5e06c2b8561c9b0244849bb7c3b5e4 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_12/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea8f46eaaa60b2c84f7d4b0e06c60bd97c046cdb4ec3c1ec67ca8bfb5f01d4a3 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_14/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_14/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..09e46bb99b0f601abe1201ccf8874db7c049ecc7 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_14/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2656f55ef62abd422318077ae274fc46b1cc1fbd3cdf6060ced0fcf27182294 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_16/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_16/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c5b5c75df4aa70985a30887d1387ea16bc93e7f --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_16/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba1f4982b3dd84c0026455749c0f75427ac4f7805325dffc25891155e1906cc9 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_18/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_18/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5df23559fc2e754435fdcbca12488472059c8dd --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_18/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6aeaa7d939c54c84dbb9a8f4a11e9b237fc2073633177408b0c6926e60fb2637 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_2/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_2/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..b4f4214a0a541d948b556e8f9cc73c6ed41d4a67 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_2/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1990f70e6a8b19f2ca7d9ecdf926a8ac00793511b5472594e86f4f65396dfe8d +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_20/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_20/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..410ef981081e89386182e8d4ee274fd6b1e2d4c0 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_20/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0763be0c8e8da79b1b147a485373b5c196e555831f6171ce5b124906cc07e9ef +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_4/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_4/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..160b085d5af2ce4912bb17c1e9c67eb4f1c575c0 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_4/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13da643585dc0c1265ba0a620a480a0f21ba2cbdcf7c5856168a7a0f6a36f368 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_6/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_6/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..4c5c63a54a97ff3ca3f8c5e15e03284117d48157 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_6/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:212b6b9542410389c51d511f4c78d57998c6f20771cb33cd884a0206107bb6fc +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_8/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_8/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1564b5231153c3e0824b6db5915f96d5c8f35a3 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_epoch_8/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7a4440f48fd341bc21a0f299a8a9a719e3442a388ed621da8bacceb7182cc36 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_final/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_final/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..410ef981081e89386182e8d4ee274fd6b1e2d4c0 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/lightweight_checkpoint_final/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0763be0c8e8da79b1b147a485373b5c196e555831f6171ce5b124906cc07e9ef +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/model-00001-of-00003.safetensors b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/model-00001-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a70cf544ea82b645a9084d345203e3b855c474af --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8b9704ab09cdeb097aa4a0a24bca96f906eec36bad63ab495bc21475058601b +size 4976698672 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/model-00002-of-00003.safetensors b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/model-00002-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..440f54a28d3e777128cca95168dd93017e99d8ef --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c28b25e7541751056ee126627e007f8d4288319733285e9f7b17b9ff6eb313f0 +size 4999802720 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/model-00003-of-00003.safetensors b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/model-00003-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e440830ec7a59efe7177c4a3bd8c895064bb568e --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2e9801c4d0865ede658e0516b6ee1cdb984bc1e449581c0b73f372904f8cac7 +size 4339154792 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/model.safetensors.index.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..2f4e99f3e7b90133b5828f4815b7834aa4245238 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/model.safetensors.index.json @@ -0,0 +1,263 @@ +{ + "metadata": { + "total_parameters": 7157813248, + "total_size": 14315626496 + }, + "weight_map": { + "lm_head.weight": "model-00003-of-00003.safetensors", + "model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.norm.weight": "model-00003-of-00003.safetensors" + } +} diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/run_args.txt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/run_args.txt new file mode 100644 index 0000000000000000000000000000000000000000..c5709e9fbfaa049c0feea7710f3b1aad099b28a2 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/run_args.txt @@ -0,0 +1,9 @@ +git_commit=unknown +dataset=slimpajama +dataset_split=train +collect_batch_size=8 +train_batch_size=32 +gradient_accumulation_step=8 +target_effective_batch=256 +command: +python /workspace/here/abprune_update/compare_model/LLM-Streamline/mseloss_entry.py --model_name meta-llama/Llama-3.1-8B --output_dir /workspace/here/abprune_update/results/llama31_8b_base_streamline_slimpajama_prune4_start23 --dataset slimpajama --dataset_split train --layer_intervals 4 --best_layer 23 --cosine_num_data 300 --train_num_data 30000 --epoches 20 --batch_size 8 --collect_batch_size 8 --train_batch_size 32 --dtype bfloat16 --gradient_accumulation_step 8 --lr 1e-5 --min_lr 5e-5 --wd 1e-3 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/special_tokens_map.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..04829afa78a2d2df203ac846968db37269b01f7f --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/special_tokens_map.json @@ -0,0 +1,17 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/tokenizer.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/tokenizer_config.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3026161bf4fc56964bad68d35d36b5b815a3f716 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/tokenizer_config.json @@ -0,0 +1,2063 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/train_loss.txt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/train_loss.txt new file mode 100644 index 0000000000000000000000000000000000000000..0b9c5c6a3bc1efab95654b971e15ac6655f38c05 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune4_start23/train_loss.txt @@ -0,0 +1,188 @@ +# Train loss logged every 100 global steps +epoch=0 global_step=100 optimizer_step=12 loss=0.1166992188 +epoch=0 global_step=200 optimizer_step=24 loss=0.1166992188 +epoch=0 global_step=300 optimizer_step=37 loss=0.1186523438 +epoch=0 global_step=400 optimizer_step=49 loss=0.1127929688 +epoch=0 global_step=500 optimizer_step=62 loss=0.1113281250 +epoch=0 global_step=600 optimizer_step=74 loss=0.1069335938 +epoch=0 global_step=700 optimizer_step=87 loss=0.1079101562 +epoch=0 global_step=800 optimizer_step=99 loss=0.1030273438 +epoch=0 global_step=900 optimizer_step=112 loss=0.0961914062 +epoch=1 global_step=1000 optimizer_step=124 loss=0.0932617188 +epoch=1 global_step=1100 optimizer_step=137 loss=0.0913085938 +epoch=1 global_step=1200 optimizer_step=149 loss=0.0888671875 +epoch=1 global_step=1300 optimizer_step=162 loss=0.0874023438 +epoch=1 global_step=1400 optimizer_step=174 loss=0.0834960938 +epoch=1 global_step=1500 optimizer_step=187 loss=0.0844726562 +epoch=1 global_step=1600 optimizer_step=199 loss=0.0864257812 +epoch=1 global_step=1700 optimizer_step=212 loss=0.0854492188 +epoch=1 global_step=1800 optimizer_step=224 loss=0.0849609375 +epoch=2 global_step=1900 optimizer_step=236 loss=0.0815429688 +epoch=2 global_step=2000 optimizer_step=249 loss=0.0830078125 +epoch=2 global_step=2100 optimizer_step=261 loss=0.0820312500 +epoch=2 global_step=2200 optimizer_step=274 loss=0.0810546875 +epoch=2 global_step=2300 optimizer_step=286 loss=0.0830078125 +epoch=2 global_step=2400 optimizer_step=299 loss=0.0800781250 +epoch=2 global_step=2500 optimizer_step=311 loss=0.0825195312 +epoch=2 global_step=2600 optimizer_step=324 loss=0.0815429688 +epoch=2 global_step=2700 optimizer_step=336 loss=0.0795898438 +epoch=2 global_step=2800 optimizer_step=349 loss=0.0834960938 +epoch=3 global_step=2900 optimizer_step=361 loss=0.0815429688 +epoch=3 global_step=3000 optimizer_step=374 loss=0.0849609375 +epoch=3 global_step=3100 optimizer_step=386 loss=0.0795898438 +epoch=3 global_step=3200 optimizer_step=399 loss=0.0825195312 +epoch=3 global_step=3300 optimizer_step=411 loss=0.0800781250 +epoch=3 global_step=3400 optimizer_step=424 loss=0.0800781250 +epoch=3 global_step=3500 optimizer_step=436 loss=0.0786132812 +epoch=3 global_step=3600 optimizer_step=449 loss=0.0800781250 +epoch=3 global_step=3700 optimizer_step=461 loss=0.0786132812 +epoch=4 global_step=3800 optimizer_step=473 loss=0.0781250000 +epoch=4 global_step=3900 optimizer_step=486 loss=0.0791015625 +epoch=4 global_step=4000 optimizer_step=498 loss=0.0766601562 +epoch=4 global_step=4100 optimizer_step=511 loss=0.0844726562 +epoch=4 global_step=4200 optimizer_step=523 loss=0.0795898438 +epoch=4 global_step=4300 optimizer_step=536 loss=0.0805664062 +epoch=4 global_step=4400 optimizer_step=548 loss=0.0771484375 +epoch=4 global_step=4500 optimizer_step=561 loss=0.0825195312 +epoch=4 global_step=4600 optimizer_step=573 loss=0.0781250000 +epoch=5 global_step=4700 optimizer_step=586 loss=0.0791015625 +epoch=5 global_step=4800 optimizer_step=598 loss=0.0815429688 +epoch=5 global_step=4900 optimizer_step=611 loss=0.0786132812 +epoch=5 global_step=5000 optimizer_step=623 loss=0.0781250000 +epoch=5 global_step=5100 optimizer_step=636 loss=0.0761718750 +epoch=5 global_step=5200 optimizer_step=648 loss=0.0751953125 +epoch=5 global_step=5300 optimizer_step=661 loss=0.0771484375 +epoch=5 global_step=5400 optimizer_step=673 loss=0.0781250000 +epoch=5 global_step=5500 optimizer_step=686 loss=0.0786132812 +epoch=5 global_step=5600 optimizer_step=698 loss=0.0795898438 +epoch=6 global_step=5700 optimizer_step=710 loss=0.0834960938 +epoch=6 global_step=5800 optimizer_step=723 loss=0.0795898438 +epoch=6 global_step=5900 optimizer_step=735 loss=0.0766601562 +epoch=6 global_step=6000 optimizer_step=748 loss=0.0825195312 +epoch=6 global_step=6100 optimizer_step=760 loss=0.0776367188 +epoch=6 global_step=6200 optimizer_step=773 loss=0.0825195312 +epoch=6 global_step=6300 optimizer_step=785 loss=0.0776367188 +epoch=6 global_step=6400 optimizer_step=798 loss=0.0781250000 +epoch=6 global_step=6500 optimizer_step=810 loss=0.0771484375 +epoch=7 global_step=6600 optimizer_step=823 loss=0.0825195312 +epoch=7 global_step=6700 optimizer_step=835 loss=0.0781250000 +epoch=7 global_step=6800 optimizer_step=848 loss=0.0805664062 +epoch=7 global_step=6900 optimizer_step=860 loss=0.0786132812 +epoch=7 global_step=7000 optimizer_step=873 loss=0.0781250000 +epoch=7 global_step=7100 optimizer_step=885 loss=0.0761718750 +epoch=7 global_step=7200 optimizer_step=898 loss=0.0751953125 +epoch=7 global_step=7300 optimizer_step=910 loss=0.0751953125 +epoch=7 global_step=7400 optimizer_step=923 loss=0.0766601562 +epoch=7 global_step=7500 optimizer_step=935 loss=0.0776367188 +epoch=8 global_step=7600 optimizer_step=947 loss=0.0756835938 +epoch=8 global_step=7700 optimizer_step=960 loss=0.0791015625 +epoch=8 global_step=7800 optimizer_step=972 loss=0.0781250000 +epoch=8 global_step=7900 optimizer_step=985 loss=0.0791015625 +epoch=8 global_step=8000 optimizer_step=997 loss=0.0761718750 +epoch=8 global_step=8100 optimizer_step=1010 loss=0.0776367188 +epoch=8 global_step=8200 optimizer_step=1022 loss=0.0756835938 +epoch=8 global_step=8300 optimizer_step=1035 loss=0.0781250000 +epoch=8 global_step=8400 optimizer_step=1047 loss=0.0766601562 +epoch=9 global_step=8500 optimizer_step=1060 loss=0.0751953125 +epoch=9 global_step=8600 optimizer_step=1072 loss=0.0810546875 +epoch=9 global_step=8700 optimizer_step=1085 loss=0.0761718750 +epoch=9 global_step=8800 optimizer_step=1097 loss=0.0781250000 +epoch=9 global_step=8900 optimizer_step=1110 loss=0.0771484375 +epoch=9 global_step=9000 optimizer_step=1122 loss=0.0747070312 +epoch=9 global_step=9100 optimizer_step=1135 loss=0.0776367188 +epoch=9 global_step=9200 optimizer_step=1147 loss=0.0737304688 +epoch=9 global_step=9300 optimizer_step=1160 loss=0.0766601562 +epoch=10 global_step=9400 optimizer_step=1172 loss=0.0766601562 +epoch=10 global_step=9500 optimizer_step=1184 loss=0.0854492188 +epoch=10 global_step=9600 optimizer_step=1197 loss=0.0781250000 +epoch=10 global_step=9700 optimizer_step=1209 loss=0.0766601562 +epoch=10 global_step=9800 optimizer_step=1222 loss=0.0795898438 +epoch=10 global_step=9900 optimizer_step=1234 loss=0.0776367188 +epoch=10 global_step=10000 optimizer_step=1247 loss=0.0751953125 +epoch=10 global_step=10100 optimizer_step=1259 loss=0.0776367188 +epoch=10 global_step=10200 optimizer_step=1272 loss=0.0786132812 +epoch=10 global_step=10300 optimizer_step=1284 loss=0.0761718750 +epoch=11 global_step=10400 optimizer_step=1297 loss=0.0761718750 +epoch=11 global_step=10500 optimizer_step=1309 loss=0.0747070312 +epoch=11 global_step=10600 optimizer_step=1322 loss=0.0776367188 +epoch=11 global_step=10700 optimizer_step=1334 loss=0.0766601562 +epoch=11 global_step=10800 optimizer_step=1347 loss=0.0751953125 +epoch=11 global_step=10900 optimizer_step=1359 loss=0.0791015625 +epoch=11 global_step=11000 optimizer_step=1372 loss=0.0751953125 +epoch=11 global_step=11100 optimizer_step=1384 loss=0.0781250000 +epoch=11 global_step=11200 optimizer_step=1397 loss=0.0776367188 +epoch=12 global_step=11300 optimizer_step=1409 loss=0.0756835938 +epoch=12 global_step=11400 optimizer_step=1421 loss=0.0766601562 +epoch=12 global_step=11500 optimizer_step=1434 loss=0.0800781250 +epoch=12 global_step=11600 optimizer_step=1446 loss=0.0781250000 +epoch=12 global_step=11700 optimizer_step=1459 loss=0.0751953125 +epoch=12 global_step=11800 optimizer_step=1471 loss=0.0756835938 +epoch=12 global_step=11900 optimizer_step=1484 loss=0.0795898438 +epoch=12 global_step=12000 optimizer_step=1496 loss=0.0751953125 +epoch=12 global_step=12100 optimizer_step=1509 loss=0.0747070312 +epoch=13 global_step=12200 optimizer_step=1521 loss=0.0751953125 +epoch=13 global_step=12300 optimizer_step=1534 loss=0.0742187500 +epoch=13 global_step=12400 optimizer_step=1546 loss=0.0766601562 +epoch=13 global_step=12500 optimizer_step=1559 loss=0.0742187500 +epoch=13 global_step=12600 optimizer_step=1571 loss=0.0756835938 +epoch=13 global_step=12700 optimizer_step=1584 loss=0.0732421875 +epoch=13 global_step=12800 optimizer_step=1596 loss=0.0771484375 +epoch=13 global_step=12900 optimizer_step=1609 loss=0.0727539062 +epoch=13 global_step=13000 optimizer_step=1621 loss=0.0747070312 +epoch=13 global_step=13100 optimizer_step=1634 loss=0.0756835938 +epoch=14 global_step=13200 optimizer_step=1646 loss=0.0786132812 +epoch=14 global_step=13300 optimizer_step=1658 loss=0.0761718750 +epoch=14 global_step=13400 optimizer_step=1671 loss=0.0791015625 +epoch=14 global_step=13500 optimizer_step=1683 loss=0.0766601562 +epoch=14 global_step=13600 optimizer_step=1696 loss=0.0751953125 +epoch=14 global_step=13700 optimizer_step=1708 loss=0.0747070312 +epoch=14 global_step=13800 optimizer_step=1721 loss=0.0742187500 +epoch=14 global_step=13900 optimizer_step=1733 loss=0.0742187500 +epoch=14 global_step=14000 optimizer_step=1746 loss=0.0786132812 +epoch=15 global_step=14100 optimizer_step=1758 loss=0.0737304688 +epoch=15 global_step=14200 optimizer_step=1771 loss=0.0776367188 +epoch=15 global_step=14300 optimizer_step=1783 loss=0.0737304688 +epoch=15 global_step=14400 optimizer_step=1796 loss=0.0771484375 +epoch=15 global_step=14500 optimizer_step=1808 loss=0.0742187500 +epoch=15 global_step=14600 optimizer_step=1821 loss=0.0761718750 +epoch=15 global_step=14700 optimizer_step=1833 loss=0.0737304688 +epoch=15 global_step=14800 optimizer_step=1846 loss=0.0756835938 +epoch=15 global_step=14900 optimizer_step=1858 loss=0.0756835938 +epoch=15 global_step=15000 optimizer_step=1871 loss=0.0737304688 +epoch=16 global_step=15100 optimizer_step=1883 loss=0.0751953125 +epoch=16 global_step=15200 optimizer_step=1895 loss=0.0737304688 +epoch=16 global_step=15300 optimizer_step=1908 loss=0.0732421875 +epoch=16 global_step=15400 optimizer_step=1920 loss=0.0751953125 +epoch=16 global_step=15500 optimizer_step=1933 loss=0.0766601562 +epoch=16 global_step=15600 optimizer_step=1945 loss=0.0732421875 +epoch=16 global_step=15700 optimizer_step=1958 loss=0.0717773438 +epoch=16 global_step=15800 optimizer_step=1970 loss=0.0771484375 +epoch=16 global_step=15900 optimizer_step=1983 loss=0.0786132812 +epoch=17 global_step=16000 optimizer_step=1995 loss=0.0722656250 +epoch=17 global_step=16100 optimizer_step=2008 loss=0.0751953125 +epoch=17 global_step=16200 optimizer_step=2020 loss=0.0761718750 +epoch=17 global_step=16300 optimizer_step=2033 loss=0.0732421875 +epoch=17 global_step=16400 optimizer_step=2045 loss=0.0751953125 +epoch=17 global_step=16500 optimizer_step=2058 loss=0.0727539062 +epoch=17 global_step=16600 optimizer_step=2070 loss=0.0771484375 +epoch=17 global_step=16700 optimizer_step=2083 loss=0.0737304688 +epoch=17 global_step=16800 optimizer_step=2095 loss=0.0742187500 +epoch=18 global_step=16900 optimizer_step=2107 loss=0.0747070312 +epoch=18 global_step=17000 optimizer_step=2120 loss=0.0712890625 +epoch=18 global_step=17100 optimizer_step=2132 loss=0.0756835938 +epoch=18 global_step=17200 optimizer_step=2145 loss=0.0727539062 +epoch=18 global_step=17300 optimizer_step=2157 loss=0.0732421875 +epoch=18 global_step=17400 optimizer_step=2170 loss=0.0751953125 +epoch=18 global_step=17500 optimizer_step=2182 loss=0.0742187500 +epoch=18 global_step=17600 optimizer_step=2195 loss=0.0742187500 +epoch=18 global_step=17700 optimizer_step=2207 loss=0.0722656250 +epoch=18 global_step=17800 optimizer_step=2220 loss=0.0722656250 +epoch=19 global_step=17900 optimizer_step=2232 loss=0.0761718750 +epoch=19 global_step=18000 optimizer_step=2245 loss=0.0756835938 +epoch=19 global_step=18100 optimizer_step=2257 loss=0.0742187500 +epoch=19 global_step=18200 optimizer_step=2270 loss=0.0717773438 +epoch=19 global_step=18300 optimizer_step=2282 loss=0.0732421875 +epoch=19 global_step=18400 optimizer_step=2295 loss=0.0751953125 +epoch=19 global_step=18500 optimizer_step=2307 loss=0.0747070312 +epoch=19 global_step=18600 optimizer_step=2320 loss=0.0732421875 +epoch=19 global_step=18700 optimizer_step=2332 loss=0.0722656250 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/config.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f03f75bf5ab64dcc05552f2b24dafb6cb4b71fab --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/config.json @@ -0,0 +1,35 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 24, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 8.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "transformers_version": "4.57.6", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/generation_config.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bab13024abb71f342624ac72ec1a8140082ce13d --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/generation_config.json @@ -0,0 +1,6 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": 128001, + "transformers_version": "4.57.6" +} diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_10/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_10/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..65354d544b405cb580898190e8f7425c08df9cba --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_10/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3ed6b4af1a9af5c3e697cc817ba87850b789609afdeef92e54580a43c91506b +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_12/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_12/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..969771726c3bcde830af8af877bd3c6cb670cdcf --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_12/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:084658c05343a209c291e5474de34dd81091610c01a3174dcde0d8c8adbc31b6 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_14/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_14/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f2e0e5557a2b75e64ee97257c395df978118524 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_14/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fb310c8cf3476b95b47890281cca16be14a883f5ddf12a2135739c778b2b9d1 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_16/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_16/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f46e3f2017f37837e4caaea7f53781b10ae1458 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_16/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38a06173e1d3fa81734ec4eb5a51025d8f4444e0d2b1da0f9b30507f88f6c731 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_18/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_18/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f266976483d234f3c01d71a6c51bfc5dd2e3152 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_18/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81d008162a047db47047754d97465c043f7a4fa0bda6a0dd80b16fb72ba29ff6 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_2/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_2/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..e52727272ef43c66f167ccf678779e39deaf9178 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_2/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6181fdf6a92c356d018052613f1c81bec863cb42916cf5468f62158db988f1fb +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_20/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_20/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..497fe6ba314095551a6b57888d85003929b106e6 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_20/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0c75dc6eea48174a67e8356c9be30713347919b8187235b0df861feb60b6390 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_4/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_4/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..e547e6c34b0b415f1b6a6de09ac927a657f95f66 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_4/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fb14b3aaccf517bcc61851dbcb03b9f856fb9909636490f0ea710c441783b8f +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_6/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_6/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..3cd51a3de5995f7e48583bf7fc84763c7c8c207f --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_6/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:764e252eb0ded8d3aa74e5b0c2b76d36702897669d2872a365f5f405e635544c +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_8/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_8/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..e04d9378d3f36862e974d064939271fdcc08a587 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_epoch_8/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:343814915b3534b8073885b7dc3fb1a055b3a157c9b013dc01504030bc5aeaa7 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_final/checkpoint.pt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_final/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..497fe6ba314095551a6b57888d85003929b106e6 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/lightweight_checkpoint_final/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0c75dc6eea48174a67e8356c9be30713347919b8187235b0df861feb60b6390 +size 436228790 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/model-00001-of-00003.safetensors b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/model-00001-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a70cf544ea82b645a9084d345203e3b855c474af --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8b9704ab09cdeb097aa4a0a24bca96f906eec36bad63ab495bc21475058601b +size 4976698672 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/model-00002-of-00003.safetensors b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/model-00002-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..440f54a28d3e777128cca95168dd93017e99d8ef --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c28b25e7541751056ee126627e007f8d4288319733285e9f7b17b9ff6eb313f0 +size 4999802720 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/model-00003-of-00003.safetensors b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/model-00003-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..853c3d875fe3fd5217b29b60e766d6fefcc713e9 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bd3e7ca3dc6f70f64e738b44837a98f04dab478faafdd4d2e1ee8728e218e6c +size 2594254592 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/model.safetensors.index.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..f152f2d21865fb8f730a69b0a19256f3500d5bd8 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/model.safetensors.index.json @@ -0,0 +1,227 @@ +{ + "metadata": { + "total_parameters": 6285365248, + "total_size": 12570730496 + }, + "weight_map": { + "lm_head.weight": "model-00003-of-00003.safetensors", + "model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.norm.weight": "model-00003-of-00003.safetensors" + } +} diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/run_args.txt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/run_args.txt new file mode 100644 index 0000000000000000000000000000000000000000..03cda1bbddb56134aa6e7d1b2d35257d367681be --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/run_args.txt @@ -0,0 +1,9 @@ +git_commit=unknown +dataset=slimpajama +dataset_split=train +collect_batch_size=8 +train_batch_size=32 +gradient_accumulation_step=8 +target_effective_batch=256 +command: +python /workspace/here/abprune_update/compare_model/LLM-Streamline/mseloss_entry.py --model_name meta-llama/Llama-3.1-8B --output_dir /workspace/here/abprune_update/results/llama31_8b_base_streamline_slimpajama_prune8_start22 --dataset slimpajama --dataset_split train --layer_intervals 8 --best_layer 22 --cosine_num_data 300 --train_num_data 30000 --epoches 20 --batch_size 8 --collect_batch_size 8 --train_batch_size 32 --dtype bfloat16 --gradient_accumulation_step 8 --lr 1e-5 --min_lr 5e-5 --wd 1e-3 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/special_tokens_map.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..04829afa78a2d2df203ac846968db37269b01f7f --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/special_tokens_map.json @@ -0,0 +1,17 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/tokenizer.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/tokenizer_config.json b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3026161bf4fc56964bad68d35d36b5b815a3f716 --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/tokenizer_config.json @@ -0,0 +1,2063 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/train_loss.txt b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/train_loss.txt new file mode 100644 index 0000000000000000000000000000000000000000..de03072d02c88ff09ff00f1de8345e8b9290f34e --- /dev/null +++ b/llmstream_results/llama31_8b_base_streamline_slimpajama_prune8_start22/train_loss.txt @@ -0,0 +1,188 @@ +# Train loss logged every 100 global steps +epoch=0 global_step=100 optimizer_step=12 loss=0.4746093750 +epoch=0 global_step=200 optimizer_step=24 loss=0.4570312500 +epoch=0 global_step=300 optimizer_step=37 loss=0.4726562500 +epoch=0 global_step=400 optimizer_step=49 loss=0.4746093750 +epoch=0 global_step=500 optimizer_step=62 loss=0.4375000000 +epoch=0 global_step=600 optimizer_step=74 loss=0.4531250000 +epoch=0 global_step=700 optimizer_step=87 loss=0.4531250000 +epoch=0 global_step=800 optimizer_step=99 loss=0.4316406250 +epoch=0 global_step=900 optimizer_step=112 loss=0.4121093750 +epoch=1 global_step=1000 optimizer_step=124 loss=0.3730468750 +epoch=1 global_step=1100 optimizer_step=137 loss=0.3593750000 +epoch=1 global_step=1200 optimizer_step=149 loss=0.3515625000 +epoch=1 global_step=1300 optimizer_step=162 loss=0.3417968750 +epoch=1 global_step=1400 optimizer_step=174 loss=0.3535156250 +epoch=1 global_step=1500 optimizer_step=187 loss=0.3457031250 +epoch=1 global_step=1600 optimizer_step=199 loss=0.3398437500 +epoch=1 global_step=1700 optimizer_step=212 loss=0.3320312500 +epoch=1 global_step=1800 optimizer_step=224 loss=0.3496093750 +epoch=2 global_step=1900 optimizer_step=236 loss=0.3378906250 +epoch=2 global_step=2000 optimizer_step=249 loss=0.3300781250 +epoch=2 global_step=2100 optimizer_step=261 loss=0.3339843750 +epoch=2 global_step=2200 optimizer_step=274 loss=0.3339843750 +epoch=2 global_step=2300 optimizer_step=286 loss=0.3300781250 +epoch=2 global_step=2400 optimizer_step=299 loss=0.3281250000 +epoch=2 global_step=2500 optimizer_step=311 loss=0.3300781250 +epoch=2 global_step=2600 optimizer_step=324 loss=0.3378906250 +epoch=2 global_step=2700 optimizer_step=336 loss=0.3164062500 +epoch=2 global_step=2800 optimizer_step=349 loss=0.3281250000 +epoch=3 global_step=2900 optimizer_step=361 loss=0.3261718750 +epoch=3 global_step=3000 optimizer_step=374 loss=0.3144531250 +epoch=3 global_step=3100 optimizer_step=386 loss=0.3105468750 +epoch=3 global_step=3200 optimizer_step=399 loss=0.3320312500 +epoch=3 global_step=3300 optimizer_step=411 loss=0.3203125000 +epoch=3 global_step=3400 optimizer_step=424 loss=0.3144531250 +epoch=3 global_step=3500 optimizer_step=436 loss=0.3164062500 +epoch=3 global_step=3600 optimizer_step=449 loss=0.3242187500 +epoch=3 global_step=3700 optimizer_step=461 loss=0.3007812500 +epoch=4 global_step=3800 optimizer_step=473 loss=0.3066406250 +epoch=4 global_step=3900 optimizer_step=486 loss=0.3125000000 +epoch=4 global_step=4000 optimizer_step=498 loss=0.3144531250 +epoch=4 global_step=4100 optimizer_step=511 loss=0.3046875000 +epoch=4 global_step=4200 optimizer_step=523 loss=0.3105468750 +epoch=4 global_step=4300 optimizer_step=536 loss=0.3125000000 +epoch=4 global_step=4400 optimizer_step=548 loss=0.3066406250 +epoch=4 global_step=4500 optimizer_step=561 loss=0.3105468750 +epoch=4 global_step=4600 optimizer_step=573 loss=0.3085937500 +epoch=5 global_step=4700 optimizer_step=586 loss=0.3046875000 +epoch=5 global_step=4800 optimizer_step=598 loss=0.3183593750 +epoch=5 global_step=4900 optimizer_step=611 loss=0.3144531250 +epoch=5 global_step=5000 optimizer_step=623 loss=0.3027343750 +epoch=5 global_step=5100 optimizer_step=636 loss=0.2968750000 +epoch=5 global_step=5200 optimizer_step=648 loss=0.3105468750 +epoch=5 global_step=5300 optimizer_step=661 loss=0.2988281250 +epoch=5 global_step=5400 optimizer_step=673 loss=0.3046875000 +epoch=5 global_step=5500 optimizer_step=686 loss=0.3007812500 +epoch=5 global_step=5600 optimizer_step=698 loss=0.2968750000 +epoch=6 global_step=5700 optimizer_step=710 loss=0.3027343750 +epoch=6 global_step=5800 optimizer_step=723 loss=0.2968750000 +epoch=6 global_step=5900 optimizer_step=735 loss=0.2968750000 +epoch=6 global_step=6000 optimizer_step=748 loss=0.3066406250 +epoch=6 global_step=6100 optimizer_step=760 loss=0.2988281250 +epoch=6 global_step=6200 optimizer_step=773 loss=0.2929687500 +epoch=6 global_step=6300 optimizer_step=785 loss=0.2949218750 +epoch=6 global_step=6400 optimizer_step=798 loss=0.2988281250 +epoch=6 global_step=6500 optimizer_step=810 loss=0.2988281250 +epoch=7 global_step=6600 optimizer_step=823 loss=0.2929687500 +epoch=7 global_step=6700 optimizer_step=835 loss=0.2988281250 +epoch=7 global_step=6800 optimizer_step=848 loss=0.2949218750 +epoch=7 global_step=6900 optimizer_step=860 loss=0.2851562500 +epoch=7 global_step=7000 optimizer_step=873 loss=0.2929687500 +epoch=7 global_step=7100 optimizer_step=885 loss=0.2968750000 +epoch=7 global_step=7200 optimizer_step=898 loss=0.2929687500 +epoch=7 global_step=7300 optimizer_step=910 loss=0.3027343750 +epoch=7 global_step=7400 optimizer_step=923 loss=0.2968750000 +epoch=7 global_step=7500 optimizer_step=935 loss=0.3242187500 +epoch=8 global_step=7600 optimizer_step=947 loss=0.3085937500 +epoch=8 global_step=7700 optimizer_step=960 loss=0.2988281250 +epoch=8 global_step=7800 optimizer_step=972 loss=0.2968750000 +epoch=8 global_step=7900 optimizer_step=985 loss=0.2968750000 +epoch=8 global_step=8000 optimizer_step=997 loss=0.2832031250 +epoch=8 global_step=8100 optimizer_step=1010 loss=0.2910156250 +epoch=8 global_step=8200 optimizer_step=1022 loss=0.2968750000 +epoch=8 global_step=8300 optimizer_step=1035 loss=0.2988281250 +epoch=8 global_step=8400 optimizer_step=1047 loss=0.2890625000 +epoch=9 global_step=8500 optimizer_step=1060 loss=0.2949218750 +epoch=9 global_step=8600 optimizer_step=1072 loss=0.3007812500 +epoch=9 global_step=8700 optimizer_step=1085 loss=0.3007812500 +epoch=9 global_step=8800 optimizer_step=1097 loss=0.2949218750 +epoch=9 global_step=8900 optimizer_step=1110 loss=0.2890625000 +epoch=9 global_step=9000 optimizer_step=1122 loss=0.2929687500 +epoch=9 global_step=9100 optimizer_step=1135 loss=0.2890625000 +epoch=9 global_step=9200 optimizer_step=1147 loss=0.2890625000 +epoch=9 global_step=9300 optimizer_step=1160 loss=0.3027343750 +epoch=10 global_step=9400 optimizer_step=1172 loss=0.2949218750 +epoch=10 global_step=9500 optimizer_step=1184 loss=0.2929687500 +epoch=10 global_step=9600 optimizer_step=1197 loss=0.2929687500 +epoch=10 global_step=9700 optimizer_step=1209 loss=0.3027343750 +epoch=10 global_step=9800 optimizer_step=1222 loss=0.3046875000 +epoch=10 global_step=9900 optimizer_step=1234 loss=0.2988281250 +epoch=10 global_step=10000 optimizer_step=1247 loss=0.2890625000 +epoch=10 global_step=10100 optimizer_step=1259 loss=0.2832031250 +epoch=10 global_step=10200 optimizer_step=1272 loss=0.3046875000 +epoch=10 global_step=10300 optimizer_step=1284 loss=0.2929687500 +epoch=11 global_step=10400 optimizer_step=1297 loss=0.2910156250 +epoch=11 global_step=10500 optimizer_step=1309 loss=0.2890625000 +epoch=11 global_step=10600 optimizer_step=1322 loss=0.2890625000 +epoch=11 global_step=10700 optimizer_step=1334 loss=0.2949218750 +epoch=11 global_step=10800 optimizer_step=1347 loss=0.2949218750 +epoch=11 global_step=10900 optimizer_step=1359 loss=0.2890625000 +epoch=11 global_step=11000 optimizer_step=1372 loss=0.2890625000 +epoch=11 global_step=11100 optimizer_step=1384 loss=0.2792968750 +epoch=11 global_step=11200 optimizer_step=1397 loss=0.2890625000 +epoch=12 global_step=11300 optimizer_step=1409 loss=0.2949218750 +epoch=12 global_step=11400 optimizer_step=1421 loss=0.2812500000 +epoch=12 global_step=11500 optimizer_step=1434 loss=0.2812500000 +epoch=12 global_step=11600 optimizer_step=1446 loss=0.2890625000 +epoch=12 global_step=11700 optimizer_step=1459 loss=0.2832031250 +epoch=12 global_step=11800 optimizer_step=1471 loss=0.2851562500 +epoch=12 global_step=11900 optimizer_step=1484 loss=0.2792968750 +epoch=12 global_step=12000 optimizer_step=1496 loss=0.2832031250 +epoch=12 global_step=12100 optimizer_step=1509 loss=0.2929687500 +epoch=13 global_step=12200 optimizer_step=1521 loss=0.2871093750 +epoch=13 global_step=12300 optimizer_step=1534 loss=0.2949218750 +epoch=13 global_step=12400 optimizer_step=1546 loss=0.2929687500 +epoch=13 global_step=12500 optimizer_step=1559 loss=0.2832031250 +epoch=13 global_step=12600 optimizer_step=1571 loss=0.2968750000 +epoch=13 global_step=12700 optimizer_step=1584 loss=0.2832031250 +epoch=13 global_step=12800 optimizer_step=1596 loss=0.2929687500 +epoch=13 global_step=12900 optimizer_step=1609 loss=0.2890625000 +epoch=13 global_step=13000 optimizer_step=1621 loss=0.2871093750 +epoch=13 global_step=13100 optimizer_step=1634 loss=0.2949218750 +epoch=14 global_step=13200 optimizer_step=1646 loss=0.2890625000 +epoch=14 global_step=13300 optimizer_step=1658 loss=0.2949218750 +epoch=14 global_step=13400 optimizer_step=1671 loss=0.3007812500 +epoch=14 global_step=13500 optimizer_step=1683 loss=0.3007812500 +epoch=14 global_step=13600 optimizer_step=1696 loss=0.2890625000 +epoch=14 global_step=13700 optimizer_step=1708 loss=0.2851562500 +epoch=14 global_step=13800 optimizer_step=1721 loss=0.2812500000 +epoch=14 global_step=13900 optimizer_step=1733 loss=0.3046875000 +epoch=14 global_step=14000 optimizer_step=1746 loss=0.2832031250 +epoch=15 global_step=14100 optimizer_step=1758 loss=0.2890625000 +epoch=15 global_step=14200 optimizer_step=1771 loss=0.2773437500 +epoch=15 global_step=14300 optimizer_step=1783 loss=0.2871093750 +epoch=15 global_step=14400 optimizer_step=1796 loss=0.2890625000 +epoch=15 global_step=14500 optimizer_step=1808 loss=0.2812500000 +epoch=15 global_step=14600 optimizer_step=1821 loss=0.2890625000 +epoch=15 global_step=14700 optimizer_step=1833 loss=0.2929687500 +epoch=15 global_step=14800 optimizer_step=1846 loss=0.2714843750 +epoch=15 global_step=14900 optimizer_step=1858 loss=0.2773437500 +epoch=15 global_step=15000 optimizer_step=1871 loss=0.2851562500 +epoch=16 global_step=15100 optimizer_step=1883 loss=0.2851562500 +epoch=16 global_step=15200 optimizer_step=1895 loss=0.2851562500 +epoch=16 global_step=15300 optimizer_step=1908 loss=0.2949218750 +epoch=16 global_step=15400 optimizer_step=1920 loss=0.2832031250 +epoch=16 global_step=15500 optimizer_step=1933 loss=0.2890625000 +epoch=16 global_step=15600 optimizer_step=1945 loss=0.2851562500 +epoch=16 global_step=15700 optimizer_step=1958 loss=0.2871093750 +epoch=16 global_step=15800 optimizer_step=1970 loss=0.2792968750 +epoch=16 global_step=15900 optimizer_step=1983 loss=0.2968750000 +epoch=17 global_step=16000 optimizer_step=1995 loss=0.2773437500 +epoch=17 global_step=16100 optimizer_step=2008 loss=0.2812500000 +epoch=17 global_step=16200 optimizer_step=2020 loss=0.2890625000 +epoch=17 global_step=16300 optimizer_step=2033 loss=0.2753906250 +epoch=17 global_step=16400 optimizer_step=2045 loss=0.2812500000 +epoch=17 global_step=16500 optimizer_step=2058 loss=0.2714843750 +epoch=17 global_step=16600 optimizer_step=2070 loss=0.2812500000 +epoch=17 global_step=16700 optimizer_step=2083 loss=0.2812500000 +epoch=17 global_step=16800 optimizer_step=2095 loss=0.2812500000 +epoch=18 global_step=16900 optimizer_step=2107 loss=0.2792968750 +epoch=18 global_step=17000 optimizer_step=2120 loss=0.2890625000 +epoch=18 global_step=17100 optimizer_step=2132 loss=0.2773437500 +epoch=18 global_step=17200 optimizer_step=2145 loss=0.2871093750 +epoch=18 global_step=17300 optimizer_step=2157 loss=0.2714843750 +epoch=18 global_step=17400 optimizer_step=2170 loss=0.2792968750 +epoch=18 global_step=17500 optimizer_step=2182 loss=0.2753906250 +epoch=18 global_step=17600 optimizer_step=2195 loss=0.3046875000 +epoch=18 global_step=17700 optimizer_step=2207 loss=0.2851562500 +epoch=18 global_step=17800 optimizer_step=2220 loss=0.2812500000 +epoch=19 global_step=17900 optimizer_step=2232 loss=0.2675781250 +epoch=19 global_step=18000 optimizer_step=2245 loss=0.2812500000 +epoch=19 global_step=18100 optimizer_step=2257 loss=0.2792968750 +epoch=19 global_step=18200 optimizer_step=2270 loss=0.2753906250 +epoch=19 global_step=18300 optimizer_step=2282 loss=0.2695312500 +epoch=19 global_step=18400 optimizer_step=2295 loss=0.2851562500 +epoch=19 global_step=18500 optimizer_step=2307 loss=0.2636718750 +epoch=19 global_step=18600 optimizer_step=2320 loss=0.2851562500 +epoch=19 global_step=18700 optimizer_step=2332 loss=0.2714843750