Emma974 commited on
Commit
c306bf4
·
verified ·
1 Parent(s): 57e98d1

Add GPT-2 Small Mixed FT LOO without ArXiv

Browse files
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation_function": "gelu_new",
3
+ "architectures": [
4
+ "GPT2LMHeadModel"
5
+ ],
6
+ "attn_pdrop": 0.1,
7
+ "bos_token_id": 50256,
8
+ "dtype": "float32",
9
+ "embd_pdrop": 0.1,
10
+ "eos_token_id": 50256,
11
+ "initializer_range": 0.02,
12
+ "layer_norm_epsilon": 1e-05,
13
+ "model_type": "gpt2",
14
+ "n_ctx": 1024,
15
+ "n_embd": 768,
16
+ "n_head": 12,
17
+ "n_inner": null,
18
+ "n_layer": 12,
19
+ "n_positions": 1024,
20
+ "reorder_and_upcast_attn": false,
21
+ "resid_pdrop": 0.1,
22
+ "scale_attn_by_inverse_layer_idx": false,
23
+ "scale_attn_weights": true,
24
+ "summary_activation": null,
25
+ "summary_first_dropout": 0.1,
26
+ "summary_proj_to_labels": true,
27
+ "summary_type": "cls_index",
28
+ "summary_use_proj": true,
29
+ "task_specific_params": {
30
+ "text-generation": {
31
+ "do_sample": true,
32
+ "max_length": 50
33
+ }
34
+ },
35
+ "transformers_version": "4.57.6",
36
+ "use_cache": true,
37
+ "vocab_size": 50257
38
+ }
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 50256,
4
+ "eos_token_id": 50256,
5
+ "transformers_version": "4.57.6"
6
+ }
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e3748079aa759e41a38e7bc9ecc2f0d754b96dbcf82487378163cba91d33a721
3
+ size 497774208
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/source_dataset_stats.csv ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ num_documents_used,raw_tokens_seen,requested_token_budget,usable_token_budget,num_sequences,context_length,effective_tokens,source,hf_dataset,hf_split,target_total_sequences,target_train_sequences,target_val_sequences,available_unique_sequences,unique_train_sequences,unique_val_sequences,num_train_sequences,num_val_sequences,train_was_upsampled,val_was_upsampled,train_extra_resampled_sequences,val_extra_resampled_sequences,train_upsampling_factor,val_upsampling_factor,train_tokens,val_tokens,final_mixture_tokens
2
+ 131,512162,512000,512000,1000,512,512000,DM Mathematics,timaeus/pile-dm_mathematics,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
3
+ 16704,495307,512000,512000,1000,512,512000,EuroParl,Helsinki-NLP/europarl,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
4
+ 118,512941,512000,512000,1000,512,512000,FreeLaw,timaeus/pile-freelaw,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
5
+ 237,535171,512000,512000,1000,512,512000,GitHub,timaeus/pile-github,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
6
+ 1,1200631,512000,512000,1000,512,512000,PG-19,emozilla/pg19,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
7
+ 524,512454,512000,512000,1000,512,512000,OWT2,suolyer/pile_openwebtext2,validation,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
8
+ 1768,510522,512000,512000,1000,512,512000,PubMed Abstracts,timaeus/pile-pubmed_abstracts,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
9
+ 65,515020,512000,512000,1000,512,512000,PubMed Central,datajuicer/the-pile-pubmed-central-refined-by-data-juicer,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
10
+ 2505,509582,512000,512000,1000,512,512000,StackExchange,flax-sentence-embeddings/stackexchange_title_body_jsonl,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
11
+ 75,518416,512000,512000,1000,512,512000,Wikipedia,wikimedia/wikipedia,train,1000,900,100,1000,900,100,900,100,False,False,0,0,1.0,1.0,460800,51200,512000
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/special_tokens_map.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "pad_token": "<|endoftext|>",
5
+ "unk_token": "<|endoftext|>"
6
+ }
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/tokenizer_config.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "50256": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": true,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ }
12
+ },
13
+ "bos_token": "<|endoftext|>",
14
+ "clean_up_tokenization_spaces": false,
15
+ "eos_token": "<|endoftext|>",
16
+ "extra_special_tokens": {},
17
+ "model_max_length": 1024,
18
+ "pad_token": "<|endoftext|>",
19
+ "tokenizer_class": "GPT2Tokenizer",
20
+ "unk_token": "<|endoftext|>"
21
+ }
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/training_history.csv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ epoch,optimizer_step,cumulative_tokens_seen,optimization_loss,train_loss,val_loss,train_ppl,val_ppl,train_acc,val_acc,train_eval_tokens,val_eval_tokens,learning_rate
2
+ 1,563,4608000,2.8710845625135635,2.6182988525781377,2.6666656621366434,13.71237695797125,14.391901638045345,0.48472042088151274,0.47924279084760485,4579151,508763,5e-05
3
+ 2,1126,9216000,2.711718646155463,2.5176119115834052,2.619658354975389,12.398951487590203,13.73103164499028,0.4976972805657643,0.4855207631058076,4579151,508763,5e-05
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/training_summary.csv ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ method,model_name,hf_repo_id,hf_subfolder,held_out_target,sources,num_sources,mixture_type,seed,context_length,requested_total_max_tokens,effective_total_source_tokens,batch_size,gradient_accumulation_steps,effective_batch_size,learning_rate,weight_decay,val_split_ratio,max_epochs,final_optimizer_step,final_tokens_seen,final_train_loss,final_val_loss,final_train_ppl,final_val_ppl,final_train_acc,final_val_acc
2
+ Mixed FT Leave-One-Out,gpt2,alignment-decision-lab/robustness-model-bank,gpt2-small/Mixed_FT_LOO_without_ArXiv/model,ArXiv,DM Mathematics | EuroParl | FreeLaw | GitHub | PG-19 | OWT2 | PubMed Abstracts | PubMed Central | StackExchange | Wikipedia,10,uniform_source_mixture,42,512,5120000,5120000,8,2,16,5e-05,0.0,0.1,2,1126,9216000,2.5176119115834052,2.619658354975389,12.398951487590203,13.73103164499028,0.4976972805657643,0.4855207631058076
gpt2-small/Mixed_FT_LOO_without_ArXiv/model/vocab.json ADDED
The diff for this file is too large to render. See raw diff