Add GPT-2 Small Mixed FT LOO without ArXiv
Browse files- gpt2-small/Mixed_FT_LOO_without_ArXiv/model/config.json +38 -0
- gpt2-small/Mixed_FT_LOO_without_ArXiv/model/generation_config.json +6 -0
- gpt2-small/Mixed_FT_LOO_without_ArXiv/model/merges.txt +0 -0
- gpt2-small/Mixed_FT_LOO_without_ArXiv/model/model.safetensors +3 -0
- gpt2-small/Mixed_FT_LOO_without_ArXiv/model/source_dataset_stats.csv +11 -0
- gpt2-small/Mixed_FT_LOO_without_ArXiv/model/special_tokens_map.json +6 -0
- gpt2-small/Mixed_FT_LOO_without_ArXiv/model/tokenizer.json +0 -0
- gpt2-small/Mixed_FT_LOO_without_ArXiv/model/tokenizer_config.json +21 -0
- gpt2-small/Mixed_FT_LOO_without_ArXiv/model/training_history.csv +3 -0
- gpt2-small/Mixed_FT_LOO_without_ArXiv/model/training_summary.csv +2 -0
- gpt2-small/Mixed_FT_LOO_without_ArXiv/model/vocab.json +0 -0
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/config.json
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"activation_function": "gelu_new",
|
| 3 |
+
"architectures": [
|
| 4 |
+
"GPT2LMHeadModel"
|
| 5 |
+
],
|
| 6 |
+
"attn_pdrop": 0.1,
|
| 7 |
+
"bos_token_id": 50256,
|
| 8 |
+
"dtype": "float32",
|
| 9 |
+
"embd_pdrop": 0.1,
|
| 10 |
+
"eos_token_id": 50256,
|
| 11 |
+
"initializer_range": 0.02,
|
| 12 |
+
"layer_norm_epsilon": 1e-05,
|
| 13 |
+
"model_type": "gpt2",
|
| 14 |
+
"n_ctx": 1024,
|
| 15 |
+
"n_embd": 768,
|
| 16 |
+
"n_head": 12,
|
| 17 |
+
"n_inner": null,
|
| 18 |
+
"n_layer": 12,
|
| 19 |
+
"n_positions": 1024,
|
| 20 |
+
"reorder_and_upcast_attn": false,
|
| 21 |
+
"resid_pdrop": 0.1,
|
| 22 |
+
"scale_attn_by_inverse_layer_idx": false,
|
| 23 |
+
"scale_attn_weights": true,
|
| 24 |
+
"summary_activation": null,
|
| 25 |
+
"summary_first_dropout": 0.1,
|
| 26 |
+
"summary_proj_to_labels": true,
|
| 27 |
+
"summary_type": "cls_index",
|
| 28 |
+
"summary_use_proj": true,
|
| 29 |
+
"task_specific_params": {
|
| 30 |
+
"text-generation": {
|
| 31 |
+
"do_sample": true,
|
| 32 |
+
"max_length": 50
|
| 33 |
+
}
|
| 34 |
+
},
|
| 35 |
+
"transformers_version": "4.57.6",
|
| 36 |
+
"use_cache": true,
|
| 37 |
+
"vocab_size": 50257
|
| 38 |
+
}
|
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/generation_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 50256,
|
| 4 |
+
"eos_token_id": 50256,
|
| 5 |
+
"transformers_version": "4.57.6"
|
| 6 |
+
}
|
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/merges.txt
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e3748079aa759e41a38e7bc9ecc2f0d754b96dbcf82487378163cba91d33a721
|
| 3 |
+
size 497774208
|
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/source_dataset_stats.csv
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
num_documents_used,raw_tokens_seen,requested_token_budget,usable_token_budget,num_sequences,context_length,effective_tokens,source,hf_dataset,hf_split,target_total_sequences,target_train_sequences,target_val_sequences,available_unique_sequences,unique_train_sequences,unique_val_sequences,num_train_sequences,num_val_sequences,train_was_upsampled,val_was_upsampled,train_extra_resampled_sequences,val_extra_resampled_sequences,train_upsampling_factor,val_upsampling_factor,train_tokens,val_tokens,final_mixture_tokens
|
| 2 |
+
131,512162,512000,512000,1000,512,512000,DM Mathematics,timaeus/pile-dm_mathematics,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
|
| 3 |
+
16704,495307,512000,512000,1000,512,512000,EuroParl,Helsinki-NLP/europarl,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
|
| 4 |
+
118,512941,512000,512000,1000,512,512000,FreeLaw,timaeus/pile-freelaw,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
|
| 5 |
+
237,535171,512000,512000,1000,512,512000,GitHub,timaeus/pile-github,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
|
| 6 |
+
1,1200631,512000,512000,1000,512,512000,PG-19,emozilla/pg19,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
|
| 7 |
+
524,512454,512000,512000,1000,512,512000,OWT2,suolyer/pile_openwebtext2,validation,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
|
| 8 |
+
1768,510522,512000,512000,1000,512,512000,PubMed Abstracts,timaeus/pile-pubmed_abstracts,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
|
| 9 |
+
65,515020,512000,512000,1000,512,512000,PubMed Central,datajuicer/the-pile-pubmed-central-refined-by-data-juicer,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
|
| 10 |
+
2505,509582,512000,512000,1000,512,512000,StackExchange,flax-sentence-embeddings/stackexchange_title_body_jsonl,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
|
| 11 |
+
75,518416,512000,512000,1000,512,512000,Wikipedia,wikimedia/wikipedia,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
|
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/special_tokens_map.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": "<|endoftext|>",
|
| 3 |
+
"eos_token": "<|endoftext|>",
|
| 4 |
+
"pad_token": "<|endoftext|>",
|
| 5 |
+
"unk_token": "<|endoftext|>"
|
| 6 |
+
}
|
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/tokenizer_config.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": false,
|
| 3 |
+
"added_tokens_decoder": {
|
| 4 |
+
"50256": {
|
| 5 |
+
"content": "<|endoftext|>",
|
| 6 |
+
"lstrip": false,
|
| 7 |
+
"normalized": true,
|
| 8 |
+
"rstrip": false,
|
| 9 |
+
"single_word": false,
|
| 10 |
+
"special": true
|
| 11 |
+
}
|
| 12 |
+
},
|
| 13 |
+
"bos_token": "<|endoftext|>",
|
| 14 |
+
"clean_up_tokenization_spaces": false,
|
| 15 |
+
"eos_token": "<|endoftext|>",
|
| 16 |
+
"extra_special_tokens": {},
|
| 17 |
+
"model_max_length": 1024,
|
| 18 |
+
"pad_token": "<|endoftext|>",
|
| 19 |
+
"tokenizer_class": "GPT2Tokenizer",
|
| 20 |
+
"unk_token": "<|endoftext|>"
|
| 21 |
+
}
|
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/training_history.csv
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
epoch,optimizer_step,cumulative_tokens_seen,optimization_loss,train_loss,val_loss,train_ppl,val_ppl,train_acc,val_acc,train_eval_tokens,val_eval_tokens,learning_rate
|
| 2 |
+
1,563,4608000,2.8710845625135635,2.6182988525781377,2.6666656621366434,13.71237695797125,14.391901638045345,0.48472042088151274,0.47924279084760485,4579151,508763,5e-05
|
| 3 |
+
2,1126,9216000,2.711718646155463,2.5176119115834052,2.619658354975389,12.398951487590203,13.73103164499028,0.4976972805657643,0.4855207631058076,4579151,508763,5e-05
|
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/training_summary.csv
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
method,model_name,hf_repo_id,hf_subfolder,held_out_target,sources,num_sources,mixture_type,seed,context_length,requested_total_max_tokens,effective_total_source_tokens,batch_size,gradient_accumulation_steps,effective_batch_size,learning_rate,weight_decay,val_split_ratio,max_epochs,final_optimizer_step,final_tokens_seen,final_train_loss,final_val_loss,final_train_ppl,final_val_ppl,final_train_acc,final_val_acc
|
| 2 |
+
Mixed FT Leave-One-Out,gpt2,alignment-decision-lab/robustness-model-bank,gpt2-small/Mixed_FT_LOO_without_ArXiv/model,ArXiv,DM Mathematics | EuroParl | FreeLaw | GitHub | PG-19 | OWT2 | PubMed Abstracts | PubMed Central | StackExchange | Wikipedia,10,uniform_source_mixture,42,512,5120000,5120000,8,2,16,5e-05,0.0,0.1,2,1126,9216000,2.5176119115834052,2.619658354975389,12.398951487590203,13.73103164499028,0.4976972805657643,0.4855207631058076
|
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/vocab.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|