2026-03-19

README.md

@@ -0,0 +1,57 @@
+Made by William Convertino
+
+conda activate /work/jf381/.cache/lmr_new
+<!-- conda create -n /work/jf381/.cache python=1.15 -->
+
+conda activate /work/jf381/.cache/lmr_new_12_15
+pip install /work/jf381/code/lm-research -e ./
+
+cd /work/jf381/code/lm-research
+bash /work/jf381/code/lm-research/scripts/training/train_bash_transformer_medium_generate.sh
+
+
+# 1. Create the environment
+# -p specifies a path (instead of -n for name)
+# python=3.10 is a stable choice (Python 1.15 does not exist)
+conda create -p /work/jf381/.cache/lmr_new_1_15_dcc python=3.10 -y
+
+# 2. Activate the environment
+conda activate /work/jf381/.cache/lmr_new_1_15_h200
+
+# 3. Install the package in editable mode
+# -e comes *before* the path
+pip install -e /work/jf381/code/lm-research
+pip install evaluate
+pip install scikit-learn
+pip install rotary_embedding_torch
+
+
+## Eval reminder
+
+There is a change in bert and gpt2 codebase
+Eval file for tinygsm:
+/work/jf381/code/lm-research/scripts/training/train_medium_bash_resume.sh
+
+1, For GPT2:
+We have gpt2 tokenizer: need tcohange
+we will have some files to modify in /work/jf381/code/lm-research/scripts/training/train_medium_bash_resume.sh
+
+FST_353M has some files trained with old version resume_new
+FST_1_3B is up to date resume
+Transformer_1_3B is up to date resume
+Transformer_353M is up to date resume
+
+2, 
+
+/work/jf381/code/lm-research/scripts/training/train_medium_bash_resume_transformer_bert_prediction.sh
+
+For Bert:
+we will start from 2 gpu version of ar model trained on slim-6B
+We have bert tokenizer: need to change 
+bert_2_gpu_transformer
+bert_2_gpu_fst
+we will have sbatch version and no svatch version be careful
+cp
+cp
+
+huggingface-cli upload jasonfan/FST_code /work/jf381/code/lm-research

compare_best_meta_scores.csv

@@ -0,0 +1,10 @@
+task,A_score,B_score,diff (A - B),pct_change_vs_B (%),a_best_meta_path,b_best_meta_path
+cola,0.4373339859333126,0.5469892544665611,-0.10965526853324853,-20.047060822097382,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/cola/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/cola/best_overall/best_meta.json
+mnli,0.8317880794701987,0.8517575140091697,-0.01996943453897093,-2.344497607655499,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/mnli/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/mnli/best_overall/best_meta.json
+mrpc,0.8823529411764706,0.8848039215686274,-0.002450980392156854,-0.2770083102493065,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/mrpc/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/mrpc/best_overall/best_meta.json
+qnli,0.9082921471718836,0.9146988833974007,-0.006406736225517129,-0.7004202521512923,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/qnli/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/qnli/best_overall/best_meta.json
+qqp,0.9106356665842197,0.9125649270343804,-0.0019292604501607302,-0.21141076026561412,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/qqp/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/qqp/best_overall/best_meta.json
+rte,0.6534296028880866,0.703971119133574,-0.05054151624548742,-7.179487179487188,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/rte/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/rte/best_overall/best_meta.json
+sst2,0.9139908256880734,0.9220183486238532,-0.008027522935779796,-0.870646766169152,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/sst2/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/sst2/best_overall/best_meta.json
+stsb,0.8884168863296509,0.8950842022895813,-0.00666731595993042,-0.7448814248844695,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/stsb/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/stsb/best_overall/best_meta.json
+wnli,0.5633802816901409,0.5915492957746479,-0.028169014084507005,-4.761904761904756,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/wnli/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/wnli/best_overall/best_meta.json

compare_best_meta_scores_with_errorbar.csv

@@ -0,0 +1,10 @@
+task,A_score,B_score,diff (A - B),pct_change_vs_B (%),a_best_meta_path,b_best_meta_path,A_mean,A_std,A_stderr,A_preferred_key,A_subset_scores,A_errorbar_path,B_mean,B_std,B_stderr,B_preferred_key,B_subset_scores,B_errorbar_path
+cola,0.4373339859333126,0.5469892544665611,-0.10965526853324853,-20.047060822097382,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/cola/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/cola/best_overall/best_meta.json,0.43723614038261793,0.016534672641352464,0.0073945304023540215,matthews_correlation,"[{""subset"": ""0123"", ""score"": 0.41263145967231507, ""metrics"": {""matthews_correlation"": 0.41263145967231507}}, {""subset"": ""1234"", ""score"": 0.42916484207638106, ""metrics"": {""matthews_correlation"": 0.42916484207638106}}, {""subset"": ""0124"", ""score"": 0.4506896605178586, ""metrics"": {""matthews_correlation"": 0.4506896605178586}}, {""subset"": ""0234"", ""score"": 0.44149495160943175, ""metrics"": {""matthews_correlation"": 0.44149495160943175}}, {""subset"": ""0134"", ""score"": 0.4521997880371033, ""metrics"": {""matthews_correlation"": 0.4521997880371033}}]",/work/jf381/output/fst_353M_bert_update_2_6b/cola/cola_validation_errorbar.json,0.5470945033883458,0.012927090515014731,0.005781170628573141,matthews_correlation,"[{""subset"": ""0123"", ""score"": 0.5377046966054644, ""metrics"": {""matthews_correlation"": 0.5377046966054644}}, {""subset"": ""1234"", ""score"": 0.5356887682287294, ""metrics"": {""matthews_correlation"": 0.5356887682287294}}, {""subset"": ""0124"", ""score"": 0.5606154723332323, ""metrics"": {""matthews_correlation"": 0.5606154723332323}}, {""subset"": ""0234"", ""score"": 0.5397567476679842, ""metrics"": {""matthews_correlation"": 0.5397567476679842}}, {""subset"": ""0134"", ""score"": 0.5617068321063187, ""metrics"": {""matthews_correlation"": 0.5617068321063187}}]",/work/jf381/output/transformer_353M_bert_update_2_6b/cola/cola_validation_errorbar.json
+mnli,0.8317880794701987,0.8517575140091697,-0.01996943453897093,-2.344497607655499,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/mnli/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/mnli/best_overall/best_meta.json,0.25000015355746574,0.003585204613965686,0.001603352246014633,accuracy,"[{""subset"": ""0123"", ""score"": 0.24472412916348843, ""metrics"": {""accuracy"": 0.24472412916348843}}, {""subset"": ""1234"", ""score"": 0.2545454545454545, ""metrics"": {""accuracy"": 0.2545454545454545}}, {""subset"": ""0124"", ""score"": 0.24917365878464276, ""metrics"": {""accuracy"": 0.24917365878464276}}, {""subset"": ""0234"", ""score"": 0.25149396058486967, ""metrics"": {""accuracy"": 0.25149396058486967}}, {""subset"": ""0134"", ""score"": 0.25006356470887364, ""metrics"": {""accuracy"": 0.25006356470887364}}]",/work/jf381/output/fst_353M_bert_update_2_6b/mnli/mnli_validation_mismatched_errorbar.json,0.4032751285882137,0.0026975390448284527,0.0012063761352392546,accuracy,"[{""subset"": ""0123"", ""score"": 0.39905924230867024, ""metrics"": {""accuracy"": 0.39905924230867024}}, {""subset"": ""1234"", ""score"": 0.4050858232676415, ""metrics"": {""accuracy"": 0.4050858232676415}}, {""subset"": ""0124"", ""score"": 0.40427154843630814, ""metrics"": {""accuracy"": 0.40427154843630814}}, {""subset"": ""0234"", ""score"": 0.40572155117609665, ""metrics"": {""accuracy"": 0.40572155117609665}}, {""subset"": ""0134"", ""score"": 0.4022374777523519, ""metrics"": {""accuracy"": 0.4022374777523519}}]",/work/jf381/output/transformer_353M_bert_update_2_6b/mnli/mnli_validation_mismatched_errorbar.json
+mrpc,0.8823529411764706,0.8848039215686274,-0.002450980392156854,-0.2770083102493065,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/mrpc/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/mrpc/best_overall/best_meta.json,0.8823399185756363,0.011540893847800208,0.0051612446329580754,accuracy,"[{""subset"": ""0123"", ""score"": 0.8868501529051988, ""metrics"": {""accuracy"": 0.8868501529051988, ""f1"": 0.9164785553047404}}, {""subset"": ""1234"", ""score"": 0.8803680981595092, ""metrics"": {""accuracy"": 0.8803680981595092, ""f1"": 0.9111617312072893}}, {""subset"": ""0124"", ""score"": 0.8990825688073395, ""metrics"": {""accuracy"": 0.8990825688073395, ""f1"": 0.9274725274725275}}, {""subset"": ""0234"", ""score"": 0.8773006134969326, ""metrics"": {""accuracy"": 0.8773006134969326, ""f1"": 0.9090909090909091}}, {""subset"": ""0134"", ""score"": 0.8680981595092024, ""metrics"": {""accuracy"": 0.8680981595092024, ""f1"": 0.9020501138952164}}]",/work/jf381/output/fst_353M_bert_update_2_6b/mrpc/mrpc_validation_errorbar.json,0.884807039267556,0.006270820544877875,0.0028043962026098397,accuracy,"[{""subset"": ""0123"", ""score"": 0.8746177370030581, ""metrics"": {""accuracy"": 0.8746177370030581, ""f1"": 0.9118279569892473}}, {""subset"": ""1234"", ""score"": 0.8865030674846626, ""metrics"": {""accuracy"": 0.8865030674846626, ""f1"": 0.9197396963123644}}, {""subset"": ""0124"", ""score"": 0.8899082568807339, ""metrics"": {""accuracy"": 0.8899082568807339, ""f1"": 0.9246861924686193}}, {""subset"": ""0234"", ""score"": 0.8834355828220859, ""metrics"": {""accuracy"": 0.8834355828220859, ""f1"": 0.9173913043478261}}, {""subset"": ""0134"", ""score"": 0.8895705521472392, ""metrics"": {""accuracy"": 0.8895705521472392, ""f1"": 0.9217391304347826}}]",/work/jf381/output/transformer_353M_bert_update_2_6b/mrpc/mrpc_validation_errorbar.json
+qnli,0.9082921471718836,0.9146988833974007,-0.006406736225517129,-0.7004202521512923,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/qnli/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/qnli/best_overall/best_meta.json,0.9082921711488293,0.002614950334436377,0.0011694413411171096,accuracy,"[{""subset"": ""0123"", ""score"": 0.9066575154426905, ""metrics"": {""accuracy"": 0.9066575154426905}}, {""subset"": ""1234"", ""score"": 0.9123569794050344, ""metrics"": {""accuracy"": 0.9123569794050344}}, {""subset"": ""0124"", ""score"": 0.9094028826355525, ""metrics"": {""accuracy"": 0.9094028826355525}}, {""subset"": ""0234"", ""score"": 0.905949656750572, ""metrics"": {""accuracy"": 0.905949656750572}}, {""subset"": ""0134"", ""score"": 0.9070938215102975, ""metrics"": {""accuracy"": 0.9070938215102975}}]",/work/jf381/output/fst_353M_bert_update_2_6b/qnli/qnli_validation_errorbar.json,0.9146989598073846,0.0013418083777259888,0.0006000749490748051,accuracy,"[{""subset"": ""0123"", ""score"": 0.9142072752230611, ""metrics"": {""accuracy"": 0.9142072752230611}}, {""subset"": ""1234"", ""score"": 0.9169336384439359, ""metrics"": {""accuracy"": 0.9169336384439359}}, {""subset"": ""0124"", ""score"": 0.9135209334248455, ""metrics"": {""accuracy"": 0.9135209334248455}}, {""subset"": ""0234"", ""score"": 0.9148741418764302, ""metrics"": {""accuracy"": 0.9148741418764302}}, {""subset"": ""0134"", ""score"": 0.9139588100686499, ""metrics"": {""accuracy"": 0.9139588100686499}}]",/work/jf381/output/transformer_353M_bert_update_2_6b/qnli/qnli_validation_errorbar.json
+qqp,0.9106356665842197,0.9125649270343804,-0.0019292604501607302,-0.21141076026561412,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/qqp/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/qqp/best_overall/best_meta.json,0.9106356665842197,0.0007774417558644825,0.0003476825229319557,accuracy,"[{""subset"": ""0123"", ""score"": 0.9109262923571605, ""metrics"": {""accuracy"": 0.9109262923571605, ""f1"": 0.8796222788618225}}, {""subset"": ""1234"", ""score"": 0.9095659163987139, ""metrics"": {""accuracy"": 0.9095659163987139, ""f1"": 0.8785652011458464}}, {""subset"": ""0124"", ""score"": 0.9100915162008409, ""metrics"": {""accuracy"": 0.9100915162008409, ""f1"": 0.8783365408752406}}, {""subset"": ""0234"", ""score"": 0.9112045510759337, ""metrics"": {""accuracy"": 0.9112045510759337, ""f1"": 0.880243515970311}}, {""subset"": ""0134"", ""score"": 0.9113900568884492, ""metrics"": {""accuracy"": 0.9113900568884492, ""f1"": 0.8804039392421966}}]",/work/jf381/output/fst_353M_bert_update_2_6b/qqp/qqp_validation_errorbar.json,0.9125649270343803,0.0011361968108467224,0.0005081226609743482,accuracy,"[{""subset"": ""0123"", ""score"": 0.9127813504823151, ""metrics"": {""accuracy"": 0.9127813504823151, ""f1"": 0.8823553943033488}}, {""subset"": ""1234"", ""score"": 0.9127504328468958, ""metrics"": {""accuracy"": 0.9127504328468958, ""f1"": 0.8830113589254622}}, {""subset"": ""0124"", ""score"": 0.9107098689092258, ""metrics"": {""accuracy"": 0.9107098689092258, ""f1"": 0.8793348374697084}}, {""subset"": ""0234"", ""score"": 0.9138325500865694, ""metrics"": {""accuracy"": 0.9138325500865694, ""f1"": 0.8841019669813283}}, {""subset"": ""0134"", ""score"": 0.9127504328468958, ""metrics"": {""accuracy"": 0.9127504328468958, ""f1"": 0.8823480363545402}}]",/work/jf381/output/transformer_353M_bert_update_2_6b/qqp/qqp_validation_errorbar.json
+rte,0.6534296028880866,0.703971119133574,-0.05054151624548742,-7.179487179487188,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/rte/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/rte/best_overall/best_meta.json,0.6534140475316945,0.01517683890482677,0.006787288694951224,accuracy,"[{""subset"": ""0123"", ""score"": 0.6396396396396397, ""metrics"": {""accuracy"": 0.6396396396396397}}, {""subset"": ""1234"", ""score"": 0.6380090497737556, ""metrics"": {""accuracy"": 0.6380090497737556}}, {""subset"": ""0124"", ""score"": 0.6666666666666666, ""metrics"": {""accuracy"": 0.6666666666666666}}, {""subset"": ""0234"", ""score"": 0.6515837104072398, ""metrics"": {""accuracy"": 0.6515837104072398}}, {""subset"": ""0134"", ""score"": 0.6711711711711712, ""metrics"": {""accuracy"": 0.6711711711711712}}]",/work/jf381/output/fst_353M_bert_update_2_6b/rte/rte_validation_errorbar.json,0.7039664098487627,0.010059305253935755,0.004498658070844226,accuracy,"[{""subset"": ""0123"", ""score"": 0.7072072072072072, ""metrics"": {""accuracy"": 0.7072072072072072}}, {""subset"": ""1234"", ""score"": 0.7149321266968326, ""metrics"": {""accuracy"": 0.7149321266968326}}, {""subset"": ""0124"", ""score"": 0.7072072072072072, ""metrics"": {""accuracy"": 0.7072072072072072}}, {""subset"": ""0234"", ""score"": 0.6877828054298643, ""metrics"": {""accuracy"": 0.6877828054298643}}, {""subset"": ""0134"", ""score"": 0.7027027027027027, ""metrics"": {""accuracy"": 0.7027027027027027}}]",/work/jf381/output/transformer_353M_bert_update_2_6b/rte/rte_validation_errorbar.json
+sst2,0.9139908256880734,0.9220183486238532,-0.008027522935779796,-0.870646766169152,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/sst2/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/sst2/best_overall/best_meta.json,0.9139911943532043,0.0020071289170888068,0.0008976153396432222,accuracy,"[{""subset"": ""0123"", ""score"": 0.9154727793696275, ""metrics"": {""accuracy"": 0.9154727793696275}}, {""subset"": ""1234"", ""score"": 0.9167862266857962, ""metrics"": {""accuracy"": 0.9167862266857962}}, {""subset"": ""0124"", ""score"": 0.9126074498567335, ""metrics"": {""accuracy"": 0.9126074498567335}}, {""subset"": ""0234"", ""score"": 0.9124820659971306, ""metrics"": {""accuracy"": 0.9124820659971306}}, {""subset"": ""0134"", ""score"": 0.9126074498567335, ""metrics"": {""accuracy"": 0.9126074498567335}}]",/work/jf381/output/fst_353M_bert_update_2_6b/sst2/sst2_validation_errorbar.json,0.922014528083929,0.006973556360779441,0.0031186692135257753,accuracy,"[{""subset"": ""0123"", ""score"": 0.9283667621776505, ""metrics"": {""accuracy"": 0.9283667621776505}}, {""subset"": ""1234"", ""score"": 0.9139167862266858, ""metrics"": {""accuracy"": 0.9139167862266858}}, {""subset"": ""0124"", ""score"": 0.9297994269340975, ""metrics"": {""accuracy"": 0.9297994269340975}}, {""subset"": ""0234"", ""score"": 0.9167862266857962, ""metrics"": {""accuracy"": 0.9167862266857962}}, {""subset"": ""0134"", ""score"": 0.9212034383954155, ""metrics"": {""accuracy"": 0.9212034383954155}}]",/work/jf381/output/transformer_353M_bert_update_2_6b/sst2/sst2_validation_errorbar.json
+stsb,0.8884168863296509,0.8950842022895813,-0.00666731595993042,-0.7448814248844695,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/stsb/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/stsb/best_overall/best_meta.json,0.8869471549987793,0.010171180366085176,0.00454869014199553,pearson,"[{""subset"": ""0123"", ""score"": 0.888052761554718, ""metrics"": {""pearson"": 0.888052761554718, ""spearmanr"": 0.8856866148079439}}, {""subset"": ""1234"", ""score"": 0.8731978535652161, ""metrics"": {""pearson"": 0.8731978535652161, ""spearmanr"": 0.871382413593779}}, {""subset"": ""0124"", ""score"": 0.8881785273551941, ""metrics"": {""pearson"": 0.8881785273551941, ""spearmanr"": 0.8866708506067439}}, {""subset"": ""0234"", ""score"": 0.8837958574295044, ""metrics"": {""pearson"": 0.8837958574295044, ""spearmanr"": 0.8797139411701278}}, {""subset"": ""0134"", ""score"": 0.9015107750892639, ""metrics"": {""pearson"": 0.9015107750892639, ""spearmanr"": 0.8941625328792709}}]",/work/jf381/output/fst_353M_bert_update_2_6b/stsb/stsb_validation_errorbar.json,-0.894455075263977,0.01112475714829985,0.004975142643355035,pearson,"[{""subset"": ""0123"", ""score"": -0.8973196148872375, ""metrics"": {""pearson"": -0.8973196148872375, ""spearmanr"": -0.8958311072773355}}, {""subset"": ""1234"", ""score"": -0.8786268830299377, ""metrics"": {""pearson"": -0.8786268830299377, ""spearmanr"": -0.8773480591609852}}, {""subset"": ""0124"", ""score"": -0.8955807089805603, ""metrics"": {""pearson"": -0.8955807089805603, ""spearmanr"": -0.8953240989573993}}, {""subset"": ""0234"", ""score"": -0.8912920355796814, ""metrics"": {""pearson"": -0.8912920355796814, ""spearmanr"": -0.8886127769347543}}, {""subset"": ""0134"", ""score"": -0.9094561338424683, ""metrics"": {""pearson"": -0.9094561338424683, ""spearmanr"": -0.9023607591347991}}]",/work/jf381/output/transformer_353M_bert_update_2_6b/stsb/stsb_validation_errorbar.json
+wnli,0.5633802816901409,0.5915492957746479,-0.028169014084507005,-4.761904761904756,/work/jf381/output/fst_353M_bert_update_2_6b/checkpoints/wnli/best_overall/best_meta.json,/work/jf381/output/transformer_353M_bert_update_2_6b/checkpoints/wnli/best_overall/best_meta.json,0.5633458646616541,0.015538973441155212,0.0069492401829973766,accuracy,"[{""subset"": ""0123"", ""score"": 0.5789473684210527, ""metrics"": {""accuracy"": 0.5789473684210527}}, {""subset"": ""1234"", ""score"": 0.5535714285714286, ""metrics"": {""accuracy"": 0.5535714285714286}}, {""subset"": ""0124"", ""score"": 0.5614035087719298, ""metrics"": {""accuracy"": 0.5614035087719298}}, {""subset"": ""0234"", ""score"": 0.543859649122807, ""metrics"": {""accuracy"": 0.543859649122807}}, {""subset"": ""0134"", ""score"": 0.5789473684210527, ""metrics"": {""accuracy"": 0.5789473684210527}}]",/work/jf381/output/fst_353M_bert_update_2_6b/wnli/wnli_validation_errorbar.json,0.5915413533834586,0.022824998420284178,0.010207649610816147,accuracy,"[{""subset"": ""0123"", ""score"": 0.6140350877192983, ""metrics"": {""accuracy"": 0.6140350877192983}}, {""subset"": ""1234"", ""score"": 0.5892857142857143, ""metrics"": {""accuracy"": 0.5892857142857143}}, {""subset"": ""0124"", ""score"": 0.6140350877192983, ""metrics"": {""accuracy"": 0.6140350877192983}}, {""subset"": ""0234"", ""score"": 0.5614035087719298, ""metrics"": {""accuracy"": 0.5614035087719298}}, {""subset"": ""0134"", ""score"": 0.5789473684210527, ""metrics"": {""accuracy"": 0.5789473684210527}}]",/work/jf381/output/transformer_353M_bert_update_2_6b/wnli/wnli_validation_errorbar.json

config/benchmark/base.yaml

@@ -0,0 +1,39 @@
+checkpoint_mode: best # Options: best, recent, epoch_x
+
+batch_size: 32
+num_workers: 8
+
+precision: bfloat16
+compile: true
+compile_mode: default
+
+prefix: null
+num_fewshot: null
+
+# Benchmark tasks to run
+tasks:
+  - hellaswag
+  - winogrande
+  - piqa
+  # - triviaqa
+  # - truthfulqa 
+  
+  # - lambada
+  - lambada_openai
+
+  - arc_challenge
+  - arc_easy
+
+  # - glue
+  # - wikitext
+  # - gsm8k
+  
+
+# Evaluation settings
+limit: 5000 # Limit number of examples per task (null for all)
+bootstrap_iters: 200 # Number of bootstrap iterations for confidence intervals
+
+# limit: 5
+# bootstrap_iters: 2
+
+use_pretrained_model: null

config/config.yaml

@@ -0,0 +1,19 @@
+defaults:
+  - model: transformer
+  - size: 1_3b
+  - training: 1_3b
+  - dataset: slimpajama_5m
+  - benchmark: base
+  - generation: base
+  - _self_
+
+hydra:
+  output_subdir: null
+  run:
+    dir: .
+
+mode: train
+checkpoint_name: auto     # By default, sets the name to {model_name}_{size_name}
+# tokenizer_base: bert-base-uncased      # Defaults to GPT2 (50k vocab size)
+tokenizer_base: gpt2     # Defaults to GPT2 (50k vocab size)
+seed: 42

config/config_bert.yaml

@@ -0,0 +1,19 @@
+defaults:
+  - model: transformer
+  - size: 1_3b
+  - training: 1_3b
+  - dataset: slimpajama_5m
+  - benchmark: base
+  - generation: base
+  - _self_
+
+hydra:
+  output_subdir: null
+  run:
+    dir: .
+
+mode: train
+checkpoint_name: auto     # By default, sets the name to {model_name}_{size_name}
+tokenizer_base: bert-base-uncased      # Defaults to GPT2 (50k vocab size)
+# tokenizer_base: gpt2     # Defaults to GPT2 (50k vocab size)
+seed: 42

config/config_gpt2.yaml

@@ -0,0 +1,19 @@
+defaults:
+  - model: transformer
+  - size: 1_3b
+  - training: 1_3b
+  - dataset: slimpajama_5m
+  - benchmark: base
+  - generation: base
+  - _self_
+
+hydra:
+  output_subdir: null
+  run:
+    dir: .
+
+mode: train
+checkpoint_name: auto     # By default, sets the name to {model_name}_{size_name}
+# tokenizer_base: bert-base-uncased      # Defaults to GPT2 (50k vocab size)
+tokenizer_base: gpt2     # Defaults to GPT2 (50k vocab size)
+seed: 42

config/dataset copy/.yaml

@@ -0,0 +1,26 @@
+# TinyGSM Dataset Configuration
+# Math reasoning dataset for language model training
+
+defaults:
+  - _base
+  - _self_
+
+# Dataset Identity
+dataset_name: tinygsm
+init_fn: lmr.data.tinygsm.initialize_dataset
+
+# Token Limits
+max_tokens_train: None       # 50M tokens for training
+max_tokens_validation: None  # 500K tokens for validation
+max_tokens_test: None        # 500K tokens for test
+
+tokens_buffer: 10k           # Additional tokens buffer to prevent inconsistencies
+
+# TinyGSM has single component (math problems)
+sampling_type: single        # Not proportional, single component
+component_name: tinygsm
+
+# HuggingFace dataset path
+hf_dataset_path: "TinyGSM/TinyGSM"  # or your TinyGSM dataset path
+training:
+  resume_checkpoint_path: /work/jf381/for_jay/best_epoch_000_step_000151000_tokens_130b_val.pt

config/dataset copy/bert_pretrain.yaml

@@ -0,0 +1,26 @@
+# TinyGSM Dataset Configuration
+# Math reasoning dataset for language model training
+
+defaults:
+  - _base
+  - _self_
+
+# Dataset Identity
+dataset_name: bert_pretrain
+init_fn: lmr.data.bert_pretrain.initialize_dataset
+
+# Token Limits
+max_tokens_train: None       # 50M tokens for training
+max_tokens_validation: None  # 500K tokens for validation
+max_tokens_test: None        # 500K tokens for test
+
+tokens_buffer: 10k           # Additional tokens buffer to prevent inconsistencies
+
+# TinyGSM has single component (math problems)
+sampling_type: single        # Not proportional, single component
+component_name: bert_pretrain
+
+# # HuggingFace dataset path
+# hf_dataset_path: "TinyGSM/TinyGSM"  # or your TinyGSM dataset path
+# training:
+#   resume_checkpoint_path: /work/jf381/for_jay/best_epoch_000_step_000151000_tokens_130b_val.pt

config/dataset copy/slimpajama_6b copy.yaml

@@ -0,0 +1,25 @@
+defaults:
+  - _base
+  - _self_
+  
+# Dataset Identity
+dataset_name: slimpajama_6b
+init_fn: lmr.data.slimpajama.initialize_dataset
+
+# Token Limits
+max_tokens_train: 6b       # Determines the number of tokens SAVED, not necessarily the total training tokens.
+max_tokens_validation: 60m
+max_tokens_test: 60m
+
+tokens_buffer: 100k      # Saves additional tokens (per component) to prevent batch/sequence level inconsistencies.
+
+# Proportional Sampling (Keeping default proportions)
+sampling_type: proportional
+proportions:
+  RedPajamaCommonCrawl: 0.522
+  RedPajamaC4: 0.267
+  RedPajamaGithub: 0.052
+  RedPajamaBook: 0.042
+  RedPajamaArXiv: 0.046
+  RedPajamaWikipedia: 0.038
+  RedPajamaStackExchange: 0.033

config/dataset copy/slimpajama_6b.yaml

@@ -0,0 +1,25 @@
+defaults:
+  - _base
+  - _self_
+  
+# Dataset Identity
+dataset_name: slimpajama_6b
+init_fn: lmr.data.slimpajama.initialize_dataset
+
+# Token Limits
+max_tokens_train: 6b       # Determines the number of tokens SAVED, not necessarily the total training tokens.
+max_tokens_validation: 60m
+max_tokens_test: 60m
+
+tokens_buffer: 100k      # Saves additional tokens (per component) to prevent batch/sequence level inconsistencies.
+
+# Proportional Sampling (Keeping default proportions)
+sampling_type: proportional
+proportions:
+  RedPajamaCommonCrawl: 0.522
+  RedPajamaC4: 0.267
+  RedPajamaGithub: 0.052
+  RedPajamaBook: 0.042
+  RedPajamaArXiv: 0.046
+  RedPajamaWikipedia: 0.038
+  RedPajamaStackExchange: 0.033

config/dataset copy/tinygsm_no_slides.yaml

@@ -0,0 +1,24 @@
+# TinyGSM Dataset Configuration
+# Math reasoning dataset for language model training
+
+defaults:
+  - _base
+  - _self_
+
+# Dataset Identity
+dataset_name: tinygsm_no_slides
+init_fn: lmr.data.tinygsm_no_slides.initialize_dataset
+
+# Token Limits
+max_tokens_train: None       # 50M tokens for training
+max_tokens_validation: None  # 500K tokens for validation
+max_tokens_test: None        # 500K tokens for test
+
+tokens_buffer: 10k           # Additional tokens buffer to prevent inconsistencies
+
+# TinyGSM has single component (math problems)
+sampling_type: single        # Not proportional, single component
+component_name: tinygsm_no_slides
+
+# HuggingFace dataset path
+hf_dataset_path: "TinyGSM/TinyGSM"  # or your TinyGSM dataset path

config/dataset copy/tinygsm_no_slides_bert.yaml

@@ -0,0 +1,26 @@
+# TinyGSM Dataset Configuration
+# Math reasoning dataset for language model training
+
+defaults:
+  - _base
+  - _self_
+
+# Dataset Identity
+dataset_name: tinygsm_no_slides_bert
+init_fn: lmr.data.tinygsm_no_slides_bert.initialize_dataset
+
+# Token Limits
+max_tokens_train: None       # 50M tokens for training
+max_tokens_validation: None  # 500K tokens for validation
+max_tokens_test: None        # 500K tokens for test
+
+tokens_buffer: 10k           # Additional tokens buffer to prevent inconsistencies
+
+# TinyGSM has single component (math problems)
+sampling_type: single        # Not proportional, single component
+component_name: tinygsm_no_slides
+
+# HuggingFace dataset path
+hf_dataset_path: "TinyGSM/TinyGSM"  # or your TinyGSM dataset path
+training:
+  resume_checkpoint_path: /work/jf381/for_jay/best_epoch_000_step_000151000_tokens_130b_val.pt

config/dataset copy/tinygsm_no_slides_resume.yaml

@@ -0,0 +1,26 @@
+# TinyGSM Dataset Configuration
+# Math reasoning dataset for language model training
+
+defaults:
+  - _base
+  - _self_
+
+# Dataset Identity
+dataset_name: tinygsm_no_slides
+init_fn: lmr.data.tinygsm_no_slides.initialize_dataset
+
+# Token Limits
+max_tokens_train: None       # 50M tokens for training
+max_tokens_validation: None  # 500K tokens for validation
+max_tokens_test: None        # 500K tokens for test
+
+tokens_buffer: 10k           # Additional tokens buffer to prevent inconsistencies
+
+# TinyGSM has single component (math problems)
+sampling_type: single        # Not proportional, single component
+component_name: tinygsm_no_slides
+
+# HuggingFace dataset path
+hf_dataset_path: "TinyGSM/TinyGSM"  # or your TinyGSM dataset path
+training:
+  resume_checkpoint_path: /work/jf381/for_jay/best_epoch_000_step_000151000_tokens_130b_val.pt

config/dataset/.yaml

@@ -0,0 +1,26 @@
+# TinyGSM Dataset Configuration
+# Math reasoning dataset for language model training
+
+defaults:
+  - _base
+  - _self_
+
+# Dataset Identity
+dataset_name: tinygsm
+init_fn: lmr.data.tinygsm.initialize_dataset
+
+# Token Limits
+max_tokens_train: None       # 50M tokens for training
+max_tokens_validation: None  # 500K tokens for validation
+max_tokens_test: None        # 500K tokens for test
+
+tokens_buffer: 10k           # Additional tokens buffer to prevent inconsistencies
+
+# TinyGSM has single component (math problems)
+sampling_type: single        # Not proportional, single component
+component_name: tinygsm
+
+# HuggingFace dataset path
+hf_dataset_path: "TinyGSM/TinyGSM"  # or your TinyGSM dataset path
+training:
+  resume_checkpoint_path: /work/jf381/for_jay/best_epoch_000_step_000151000_tokens_130b_val.pt

config/dataset/_base.yaml

@@ -0,0 +1,5 @@
+# Sampling
+sampling_type: null
+
+# Misc
+component_whitelist: null

config/dataset/slimpajama_120b.yaml

@@ -0,0 +1,25 @@
+defaults:
+  - _base
+  - _self_
+  
+# Dataset Identity
+dataset_name: slimpajama_120b
+init_fn: lmr.data.slimpajama.initialize_dataset
+
+# Token Limits
+max_tokens_train: 120b       # Determines the number of tokens SAVED, not necessarily the total training tokens.
+max_tokens_validation: 60m
+max_tokens_test: 60m
+
+tokens_buffer: 100k          # Saves additional tokens (per component) to prevent batch/sequence level inconsistencies.
+
+# Proportional Sampling (Keeping default proportions)
+sampling_type: proportional
+proportions:
+  RedPajamaCommonCrawl: 0.522
+  RedPajamaC4: 0.267
+  RedPajamaGithub: 0.052
+  RedPajamaBook: 0.042
+  RedPajamaArXiv: 0.046
+  RedPajamaWikipedia: 0.038
+  RedPajamaStackExchange: 0.033

config/dataset/slimpajama_5m.yaml

@@ -0,0 +1,25 @@
+defaults:
+  - _base
+  - _self_
+  
+# Dataset Identity
+dataset_name: slimpajama_5m
+init_fn: lmr.data.slimpajama.initialize_dataset
+
+# Token Limits
+max_tokens_train: 5m
+max_tokens_validation: 1m
+max_tokens_test: 1m
+
+tokens_buffer: 10k
+
+# Proportional Sampling
+sampling_type: proportional
+proportions:
+  RedPajamaCommonCrawl: 0.38
+  RedPajamaC4: 0.20
+  RedPajamaGithub: 0.06
+  RedPajamaBook: 0.09
+  RedPajamaArXiv: 0.09
+  RedPajamaWikipedia: 0.09
+  RedPajamaStackExchange: 0.09

config/dataset/slimpajama_60b.yaml

@@ -0,0 +1,25 @@
+defaults:
+  - _base
+  - _self_
+  
+# Dataset Identity
+dataset_name: slimpajama_60b
+init_fn: lmr.data.slimpajama.initialize_dataset
+
+# Token Limits
+max_tokens_train: 60b       # Determines the number of tokens SAVED, not necessarily the total training tokens.
+max_tokens_validation: 60m
+max_tokens_test: 60m
+
+tokens_buffer: 100k          # Saves additional tokens (per component) to prevent batch/sequence level inconsistencies.
+
+# Proportional Sampling (Keeping default proportions)
+sampling_type: proportional
+proportions:
+  RedPajamaCommonCrawl: 0.522
+  RedPajamaC4: 0.267
+  RedPajamaGithub: 0.052
+  RedPajamaBook: 0.042
+  RedPajamaArXiv: 0.046
+  RedPajamaWikipedia: 0.038
+  RedPajamaStackExchange: 0.033

config/dataset/slimpajama_6b copy.yaml

@@ -0,0 +1,25 @@
+defaults:
+  - _base
+  - _self_
+  
+# Dataset Identity
+dataset_name: slimpajama_6b
+init_fn: lmr.data.slimpajama.initialize_dataset
+
+# Token Limits
+max_tokens_train: 6b       # Determines the number of tokens SAVED, not necessarily the total training tokens.
+max_tokens_validation: 60m
+max_tokens_test: 60m
+
+tokens_buffer: 100k      # Saves additional tokens (per component) to prevent batch/sequence level inconsistencies.
+
+# Proportional Sampling (Keeping default proportions)
+sampling_type: proportional
+proportions:
+  RedPajamaCommonCrawl: 0.522
+  RedPajamaC4: 0.267
+  RedPajamaGithub: 0.052
+  RedPajamaBook: 0.042
+  RedPajamaArXiv: 0.046
+  RedPajamaWikipedia: 0.038
+  RedPajamaStackExchange: 0.033

config/dataset/slimpajama_6b.yaml

@@ -0,0 +1,25 @@
+defaults:
+  - _base
+  - _self_
+  
+# Dataset Identity
+dataset_name: slimpajama_6b
+init_fn: lmr.data.slimpajama.initialize_dataset
+
+# Token Limits
+max_tokens_train: 6b       # Determines the number of tokens SAVED, not necessarily the total training tokens.
+max_tokens_validation: 60m
+max_tokens_test: 60m
+
+tokens_buffer: 100k      # Saves additional tokens (per component) to prevent batch/sequence level inconsistencies.
+
+# Proportional Sampling (Keeping default proportions)
+sampling_type: proportional
+proportions:
+  RedPajamaCommonCrawl: 0.522
+  RedPajamaC4: 0.267
+  RedPajamaGithub: 0.052
+  RedPajamaBook: 0.042
+  RedPajamaArXiv: 0.046
+  RedPajamaWikipedia: 0.038
+  RedPajamaStackExchange: 0.033

config/dataset/tinygsm.yaml

@@ -0,0 +1,26 @@
+# TinyGSM Dataset Configuration
+# Math reasoning dataset for language model training
+
+defaults:
+  - _base
+  - _self_
+
+# Dataset Identity
+dataset_name: tinygsm
+init_fn: lmr.data.tinygsm.initialize_dataset
+
+# Token Limits
+max_tokens_train: None       # 50M tokens for training
+max_tokens_validation: None  # 500K tokens for validation
+max_tokens_test: None        # 500K tokens for test
+
+tokens_buffer: 10k           # Additional tokens buffer to prevent inconsistencies
+
+# TinyGSM has single component (math problems)
+sampling_type: single        # Not proportional, single component
+component_name: tinygsm
+
+# HuggingFace dataset path
+hf_dataset_path: "TinyGSM/TinyGSM"  # or your TinyGSM dataset path
+training:
+  resume_checkpoint_path: /work/jf381/for_jay/best_epoch_000_step_000151000_tokens_130b_val.pt

config/dataset/tinygsm_no_slides.yaml

@@ -0,0 +1,24 @@
+# TinyGSM Dataset Configuration
+# Math reasoning dataset for language model training
+
+defaults:
+  - _base
+  - _self_
+
+# Dataset Identity
+dataset_name: tinygsm_no_slides
+init_fn: lmr.data.tinygsm_no_slides.initialize_dataset
+
+# Token Limits
+max_tokens_train: None       # 50M tokens for training
+max_tokens_validation: None  # 500K tokens for validation
+max_tokens_test: None        # 500K tokens for test
+
+tokens_buffer: 10k           # Additional tokens buffer to prevent inconsistencies
+
+# TinyGSM has single component (math problems)
+sampling_type: single        # Not proportional, single component
+component_name: tinygsm_no_slides
+
+# HuggingFace dataset path
+hf_dataset_path: "TinyGSM/TinyGSM"  # or your TinyGSM dataset path

config/dataset/tinygsm_no_slides_resume.yaml

@@ -0,0 +1,26 @@
+# TinyGSM Dataset Configuration
+# Math reasoning dataset for language model training
+
+defaults:
+  - _base
+  - _self_
+
+# Dataset Identity
+dataset_name: tinygsm_no_slides
+init_fn: lmr.data.tinygsm_no_slides.initialize_dataset
+
+# Token Limits
+max_tokens_train: None       # 50M tokens for training
+max_tokens_validation: None  # 500K tokens for validation
+max_tokens_test: None        # 500K tokens for test
+
+tokens_buffer: 10k           # Additional tokens buffer to prevent inconsistencies
+
+# TinyGSM has single component (math problems)
+sampling_type: single        # Not proportional, single component
+component_name: tinygsm_no_slides
+
+# HuggingFace dataset path
+hf_dataset_path: "TinyGSM/TinyGSM"  # or your TinyGSM dataset path
+training:
+  resume_checkpoint_path: /work/jf381/for_jay/best_epoch_000_step_000151000_tokens_130b_val.pt

config/dataset/tinygsm_resume.yaml

@@ -0,0 +1,26 @@
+# TinyGSM Dataset Configuration
+# Math reasoning dataset for language model training
+
+defaults:
+  - _base
+  - _self_
+
+# Dataset Identity
+dataset_name: tinygsm
+init_fn: lmr.data.tinygsm.initialize_dataset
+
+# Token Limits
+max_tokens_train: None       # 50M tokens for training
+max_tokens_validation: None  # 500K tokens for validation
+max_tokens_test: None        # 500K tokens for test
+
+tokens_buffer: 10k           # Additional tokens buffer to prevent inconsistencies
+
+# TinyGSM has single component (math problems)
+sampling_type: single        # Not proportional, single component
+component_name: tinygsm
+
+# HuggingFace dataset path
+hf_dataset_path: "TinyGSM/TinyGSM"  # or your TinyGSM dataset path
+training:
+  resume_checkpoint_path: /work/jf381/for_jay/best_epoch_000_step_000151000_tokens_130b_val.pt

config/generation/base.yaml

@@ -0,0 +1,7 @@
+max_new_tokens: 100
+temperature: 0.8
+top_p: 0.9
+return_generation_only: true
+
+save_to_file: true
+output_dir: outputs/generations

config/model/_base_fst.yaml

@@ -0,0 +1,45 @@
+# === Model Identity ===
+config_target: lmr.models.fst.FSTConfig
+model_target: lmr.models.fst.FSTForCausalLM
+model_name: fst
+
+# === Model Architecture ===
+vocab_size: 50257
+hidden_size: 1024
+embedding_size: null          # if null, defaults to hidden_size
+num_hidden_layers: 24
+num_attention_heads: 16
+intermediate_size: 4096
+max_position_embeddings: 1024
+tie_word_embeddings: true
+use_causal_attention: true
+share_f_and_phi_embedding: true
+
+# === FST Specific Overrides (all default to non-fst values) ===
+hidden_size_f: null
+hidden_size_phi: null
+embedding_size_f: null
+embedding_size_phi: null
+num_attention_heads_f: null
+num_attention_heads_phi: null
+intermediate_size_f: null
+intermediate_size_phi: null
+
+# === FST Experimental Settings (Not yet implemented) ===
+# num_phi_updates: 1                # Number of phi updates in a single phi layer
+# num_f_updates: 1                  # Number of f updates in a single f layer
+# duplicate_phi_updates: false      # Whether the phi updates within a layer should share weights (only used when num_phi_updates > 1)
+# duplicate_f_updates: false        # Whether to duplicate f updates within a layer (only used when num_f_updates > 1)
+
+# === Initialization ===
+initializer_range: 0.02
+
+# === Misc ===
+use_cache: false
+self.truncate_activation_size: false
+
+# === Tokenizer ===
+tokenizer_type: gpt2          # Hugging Face tokenizer name or path
+bos_token_id: null            # if null, defaults to tokenizer bos_token_id
+eos_token_id: null            # if null, defaults to tokenizer eos_token_id
+pad_token_id: null            # if null, defaults to tokenizer pad_token_id

config/model/_base_fst_bert.yaml

@@ -0,0 +1,45 @@
+# === Model Identity ===
+config_target: lmr.models.fst.FSTConfig
+model_target: lmr.models.fst.FSTForMaskedLM
+model_name: fst
+
+# === Model Architecture ===
+vocab_size: 50257
+hidden_size: 1024
+embedding_size: null          # if null, defaults to hidden_size
+num_hidden_layers: 24
+num_attention_heads: 16
+intermediate_size: 4096
+max_position_embeddings: 1024
+tie_word_embeddings: true
+use_causal_attention: false
+share_f_and_phi_embedding: true
+
+# === FST Specific Overrides (all default to non-fst values) ===
+hidden_size_f: null
+hidden_size_phi: null
+embedding_size_f: null
+embedding_size_phi: null
+num_attention_heads_f: null
+num_attention_heads_phi: null
+intermediate_size_f: null
+intermediate_size_phi: null
+
+# === FST Experimental Settings (Not yet implemented) ===
+# num_phi_updates: 1                # Number of phi updates in a single phi layer
+# num_f_updates: 1                  # Number of f updates in a single f layer
+# duplicate_phi_updates: false      # Whether the phi updates within a layer should share weights (only used when num_phi_updates > 1)
+# duplicate_f_updates: false        # Whether to duplicate f updates within a layer (only used when num_f_updates > 1)
+
+# === Initialization ===
+initializer_range: 0.02
+
+# === Misc ===
+use_cache: false
+self.truncate_activation_size: false
+
+# === Tokenizer ===
+tokenizer_type: gpt2          # Hugging Face tokenizer name or path
+bos_token_id: null            # if null, defaults to tokenizer bos_token_id
+eos_token_id: null            # if null, defaults to tokenizer eos_token_id
+pad_token_id: null            # if null, defaults to tokenizer pad_token_id

config/model/_base_fst_bert_predicition.yaml

@@ -0,0 +1,45 @@
+# === Model Identity ===
+config_target: lmr.models.fst.FSTConfig
+model_target: lmr.models.fst.FSTForSequenceClassification
+model_name: fst
+
+# === Model Architecture ===
+vocab_size: 50257
+hidden_size: 1024
+embedding_size: null          # if null, defaults to hidden_size
+num_hidden_layers: 24
+num_attention_heads: 16
+intermediate_size: 4096
+max_position_embeddings: 1024
+tie_word_embeddings: true
+use_causal_attention: false
+share_f_and_phi_embedding: true
+
+# === FST Specific Overrides (all default to non-fst values) ===
+hidden_size_f: null
+hidden_size_phi: null
+embedding_size_f: null
+embedding_size_phi: null
+num_attention_heads_f: null
+num_attention_heads_phi: null
+intermediate_size_f: null
+intermediate_size_phi: null
+
+# === FST Experimental Settings (Not yet implemented) ===
+# num_phi_updates: 1                # Number of phi updates in a single phi layer
+# num_f_updates: 1                  # Number of f updates in a single f layer
+# duplicate_phi_updates: false      # Whether the phi updates within a layer should share weights (only used when num_phi_updates > 1)
+# duplicate_f_updates: false        # Whether to duplicate f updates within a layer (only used when num_f_updates > 1)
+
+# === Initialization ===
+initializer_range: 0.02
+
+# === Misc ===
+use_cache: false
+self.truncate_activation_size: false
+
+# === Tokenizer ===
+tokenizer_type: gpt2          # Hugging Face tokenizer name or path
+bos_token_id: null            # if null, defaults to tokenizer bos_token_id
+eos_token_id: null            # if null, defaults to tokenizer eos_token_id
+pad_token_id: null            # if null, defaults to tokenizer pad_token_id

config/model/_base_fst_predicition.yaml

@@ -0,0 +1,45 @@
+# === Model Identity ===
+config_target: lmr.models.fst.FSTConfig
+model_target: lmr.models.fst.FSTForSequenceClassification
+model_name: fst
+
+# === Model Architecture ===
+vocab_size: 50257
+hidden_size: 1024
+embedding_size: null          # if null, defaults to hidden_size
+num_hidden_layers: 24
+num_attention_heads: 16
+intermediate_size: 4096
+max_position_embeddings: 1024
+tie_word_embeddings: true
+use_causal_attention: true
+share_f_and_phi_embedding: true
+
+# === FST Specific Overrides (all default to non-fst values) ===
+hidden_size_f: null
+hidden_size_phi: null
+embedding_size_f: null
+embedding_size_phi: null
+num_attention_heads_f: null
+num_attention_heads_phi: null
+intermediate_size_f: null
+intermediate_size_phi: null
+
+# === FST Experimental Settings (Not yet implemented) ===
+# num_phi_updates: 1                # Number of phi updates in a single phi layer
+# num_f_updates: 1                  # Number of f updates in a single f layer
+# duplicate_phi_updates: false      # Whether the phi updates within a layer should share weights (only used when num_phi_updates > 1)
+# duplicate_f_updates: false        # Whether to duplicate f updates within a layer (only used when num_f_updates > 1)
+
+# === Initialization ===
+initializer_range: 0.02
+
+# === Misc ===
+use_cache: false
+self.truncate_activation_size: false
+
+# === Tokenizer ===
+tokenizer_type: gpt2          # Hugging Face tokenizer name or path
+bos_token_id: null            # if null, defaults to tokenizer bos_token_id
+eos_token_id: null            # if null, defaults to tokenizer eos_token_id
+pad_token_id: null            # if null, defaults to tokenizer pad_token_id

config/model/_base_transformer_bert.yaml

@@ -0,0 +1,28 @@
+# === Model Identity ===
+config_target: lmr.models.transformer_bert.TransformerConfig
+model_target: lmr.models.transformer_bert.TransformerForMaskedLM
+model_name: transformer
+
+# === Model Architecture ===
+vocab_size: 50257
+hidden_size: 1024
+embedding_size: null          # if null, defaults to hidden_size in code
+num_hidden_layers: 24
+num_attention_heads: 16
+intermediate_size: 4096
+max_position_embeddings: 1024
+tie_word_embeddings: true
+use_causal_attention: false
+
+# === Initialization ===
+initializer_range: 0.02
+
+# === Misc ===
+use_cache: false
+self.truncate_activation_size: false
+
+# === Tokenizer ===
+tokenizer_type: bert-base-uncased          # Hugging Face tokenizer name or path
+bos_token_id: null            # if null, defaults to tokenizer bos_token_id
+eos_token_id: null            # if null, defaults to tokenizer eos_token_id
+pad_token_id: null            # if null, defaults to tokenizer pad_token_id

config/model/_base_transformer_bert_prediction.yaml

@@ -0,0 +1,28 @@
+# === Model Identity ===
+config_target: lmr.models.transformer_bert.TransformerConfig
+model_target: lmr.models.transformer_bert.TransformerForSequenceClassification
+model_name: transformer
+
+# === Model Architecture ===
+vocab_size: 50257
+hidden_size: 1024
+embedding_size: null          # if null, defaults to hidden_size in code
+num_hidden_layers: 24
+num_attention_heads: 16
+intermediate_size: 4096
+max_position_embeddings: 1024
+tie_word_embeddings: true
+use_causal_attention: false
+
+# === Initialization ===
+initializer_range: 0.02
+
+# === Misc ===
+use_cache: false
+self.truncate_activation_size: false
+
+# === Tokenizer ===
+tokenizer_type: bert-base-uncased          # Hugging Face tokenizer name or path
+bos_token_id: null            # if null, defaults to tokenizer bos_token_id
+eos_token_id: null            # if null, defaults to tokenizer eos_token_id
+pad_token_id: null            # if null, defaults to tokenizer pad_token_id

config/model/_base_transformer_prediction.yaml

@@ -0,0 +1,28 @@
+# === Model Identity ===
+config_target: lmr.models.transformer_bert.TransformerConfig
+model_target: lmr.models.transformer_bert.TransformerForSequenceClassification
+model_name: transformer
+
+# === Model Architecture ===
+vocab_size: 50257
+hidden_size: 1024
+embedding_size: null          # if null, defaults to hidden_size in code
+num_hidden_layers: 24
+num_attention_heads: 16
+intermediate_size: 4096
+max_position_embeddings: 1024
+tie_word_embeddings: true
+use_causal_attention: true
+
+# === Initialization ===
+initializer_range: 0.02
+
+# === Misc ===
+use_cache: false
+self.truncate_activation_size: false
+
+# === Tokenizer ===
+tokenizer_type: bert-base-uncased          # Hugging Face tokenizer name or path
+bos_token_id: null            # if null, defaults to tokenizer bos_token_id
+eos_token_id: null            # if null, defaults to tokenizer eos_token_id
+pad_token_id: null            # if null, defaults to tokenizer pad_token_id

config/model/fst_353M_bert copy.yaml

@@ -0,0 +1,14 @@
+defaults:
+  - _base_fst_bert
+  - _self_
+
+# === Model Identity ===
+model_name: fst_353M_bert
+
+# === Model Architecture ===
+hidden_size: 1024
+num_hidden_layers: 24
+num_attention_heads: 16
+intermediate_size: 4096
+max_position_embeddings: 1024
+max_seq_len: 1024

config/model/fst_353M_bert.yaml

@@ -0,0 +1,14 @@
+defaults:
+  - _base_fst_bert
+  - _self_
+
+# === Model Identity ===
+model_name: fst_353M_bert
+
+# === Model Architecture ===
+hidden_size: 1024
+num_hidden_layers: 24
+num_attention_heads: 16
+intermediate_size: 4096
+max_position_embeddings: 1024
+max_seq_len: 1024

config/model/fst_353M_bert_prediction.yaml

@@ -0,0 +1,14 @@
+defaults:
+  - _base_fst_bert_predicition
+  - _self_
+
+# === Model Identity ===
+model_name: fst_353M_prediction
+
+# === Model Architecture ===
+hidden_size: 1024
+num_hidden_layers: 24
+num_attention_heads: 16
+intermediate_size: 4096
+max_position_embeddings: 1024
+max_seq_len: 1024

config/model/fst_353M_prediction.yaml

@@ -0,0 +1,14 @@
+defaults:
+  - _base_fst_predicition
+  - _self_
+
+# === Model Identity ===
+model_name: fst_353M_prediction
+
+# === Model Architecture ===
+hidden_size: 1024
+num_hidden_layers: 24
+num_attention_heads: 16
+intermediate_size: 4096
+max_position_embeddings: 1024
+max_seq_len: 1024

config/model/transformer_353M.yaml

@@ -0,0 +1,14 @@
+defaults:
+  - _base_transformer
+  - _self_
+
+# === Model Identity ===
+model_name: transformer_353M
+
+# === Model Architecture ===
+hidden_size: 1024
+num_hidden_layers: 24
+num_attention_heads: 16
+intermediate_size: 4096
+max_position_embeddings: 1024
+max_seq_len: 1024

config/model/transformer_353M_bert.yaml

@@ -0,0 +1,14 @@
+defaults:
+  - _base_transformer_bert
+  - _self_
+
+# === Model Identity ===
+model_name: transformer_353M_prediction
+
+# === Model Architecture ===
+hidden_size: 1024
+num_hidden_layers: 24
+num_attention_heads: 16
+intermediate_size: 4096
+max_position_embeddings: 1024
+max_seq_len: 1024

config/model/transformer_353M_bert_prediction.yaml

@@ -0,0 +1,14 @@
+defaults:
+  - _base_transformer_bert_prediction
+  - _self_
+
+# === Model Identity ===
+model_name: transformer_353M_bert
+
+# === Model Architecture ===
+hidden_size: 1024
+num_hidden_layers: 24
+num_attention_heads: 16
+intermediate_size: 4096
+max_position_embeddings: 1024
+max_seq_len: s

config/model/transformer_353M_prediction.yaml

@@ -0,0 +1,14 @@
+defaults:
+  - _base_transformer_prediction
+  - _self_
+
+# === Model Identity ===
+model_name: transformer_353M_prediction
+
+# === Model Architecture ===
+hidden_size: 1024
+num_hidden_layers: 24
+num_attention_heads: 16
+intermediate_size: 4096
+max_position_embeddings: 1024
+max_seq_len: s

config/size/1_3b.yaml

@@ -0,0 +1,12 @@
+defaults:
+  - _base
+  - _self_
+
+# Size Identity
+size_name: 1_3b
+
+# Dimensions
+n_layers: 24
+n_heads: 32
+hidden_dim: 2048
+max_seq_len: 1024

config/size/7b.yaml

@@ -0,0 +1,12 @@
+defaults:
+  - _base
+  - _self_
+
+# Size Identity
+size_name: 7b
+
+# Dimensions
+n_layers: 32
+n_heads: 32
+hidden_dim: 4096
+max_seq_len: 4096

config/size/_base.yaml

@@ -0,0 +1,15 @@
+# Fine-tuned dimension control
+embed_dim: auto                    # Defaults to hidden_dim
+mlp_dim: auto                      # Defaults to hidden_dim × 4
+
+# FST dims
+n_heads_f: auto                    # FST: Defaults to n_heads
+n_heads_phi: auto                  # FST: Defaults to n_heads
+hidden_dim_f: auto                 # FST: Defaults to hidden_dim
+hidden_dim_phi: auto               # FST: Defaults to hidden_dim
+
+# FST fine-tuned dimension control
+embed_dim_f: auto                  # FST: Defaults to hidden_dim_f
+embed_dim_phi: auto                # FST: Defaults to hidden_dim_phi
+mlp_dim_f: auto                    # FST: Defaults to hidden_dim_f × 4
+mlp_dim_phi: auto                  # FST: Defaults to hidden_dim_phi × 4

config/size/large.yaml

@@ -0,0 +1,12 @@
+defaults:
+  - _base
+  - _self_
+
+# Size Identity
+size_name: large
+
+# Dimensions
+n_layers: 36
+n_heads: 20
+hidden_dim: 1280
+max_seq_len: 1024

config/size/medium.yaml

@@ -0,0 +1,12 @@
+defaults:
+  - _base
+  - _self_
+
+# Size Identity
+size_name: medium
+
+# Dimensions
+n_layers: 24
+n_heads: 16
+hidden_dim: 1024
+max_seq_len: 1024

config/size/medium_23.yaml

@@ -0,0 +1,18 @@
+defaults:
+  - medium
+  - _self_
+
+# Size Identity
+size_name: medium_23
+
+# Dimensions
+max_seq_len: 1024
+n_layers: 24
+n_heads: 16
+
+# FST dims
+# hidden_dim: 1024
+hidden_dim_phi: 768
+mlp_dim_phi: 3072
+hidden_dim_f: 1280
+mlp_dim_f: 5120

config/size/medium_32.yaml

@@ -0,0 +1,18 @@
+defaults:
+  - medium
+  - _self_
+
+# Size Identity
+size_name: medium_32
+
+# Dimensions
+max_seq_len: 1024
+n_layers: 24
+n_heads: 16
+
+# FST dims
+# hidden_dim: 1024
+hidden_dim_phi: 1280
+mlp_dim_phi: 5120
+hidden_dim_f: 768
+mlp_dim_f: 3072

config/size/small.yaml

@@ -0,0 +1,12 @@
+defaults:
+  - _base
+  - _self_
+
+# Size Identity
+size_name: small
+
+# Dimensions
+n_layers: 12
+n_heads: 12
+hidden_dim: 768
+max_seq_len: 1024

config/training/1_3b.yaml

@@ -0,0 +1,32 @@
+defaults:
+  - _base
+  - _self_
+
+# Identity
+training_name: 1_3b
+use_ddp: true
+
+# Training
+max_epochs: 5
+warmup_steps: 500
+
+# Precision and Compilation
+precision: bfloat16
+compile: true
+compile_mode: default
+
+# Learning Rate + LR Search
+lr: 2e-4
+
+# Data Loading
+batch_size: 16
+num_workers: 8
+use_sliding_window: true
+
+# Gradient Accumulation and Step Tracking
+use_grad_accum: true
+grad_accum_steps: auto
+tokens_per_step: 1048576     # ~1m (Note that this is not perfectly enforced and may be adjusted based on batch size and world size)
+
+# Validation Steps
+validation_steps: 1000