Olmo anchor at common step 300

chat_template.jinja

@@ -0,0 +1,16 @@
+{%- set has_system = messages|selectattr('role', 'equalto', 'system')|list|length > 0 -%}{%- if not has_system -%}{{- '<|im_start|>system
+You are a helpful function-calling AI assistant. ' -}}{%- if tools is none -%}{{- 'You do not currently have access to any functions. <functions></functions><|im_end|>
+' -}}{%- else -%}{{- 'You are provided with function signatures within <functions></functions> XML tags. You may call one or more functions to assist with the user query. Output any function calls within <function_calls></function_calls> XML tags. Do not make assumptions about what values to plug into functions.' -}}{{- '<functions>' -}}{{- tools | tojson -}}{{- '</functions><|im_end|>
+' -}}{%- endif -%}{%- endif -%}{%- for message in messages -%}{%- if message['role'] == 'system' -%}{{- '<|im_start|>system
+' + message['content'] -}}{%- if tools is not none -%}{{- '<functions>' -}}{{- tools | tojson -}}{{- '</functions>' -}}{%- elif message.get('functions', none) is not none -%}{{- ' <functions>' + message['functions'] + '</functions>' -}}{%- endif -%}{{- '<|im_end|>
+' -}}{%- elif message['role'] == 'user' -%}{{- '<|im_start|>user
+' + message['content'] + '<|im_end|>
+' -}}{%- elif message['role'] == 'assistant' -%}{{- '<|im_start|>assistant
+' -}}{%- if message.get('content', none) is not none -%}{{- message['content'] -}}{%- endif -%}{%- if message.get('function_calls', none) is not none -%}{{- '<function_calls>' + message['function_calls'] + '</function_calls>' -}}{% elif message.get('tool_calls', none) is not none %}{{- '<function_calls>' -}}{%- for tool_call in message['tool_calls'] %}{%- if tool_call is mapping and tool_call.get('function', none) is not none %}{%- set args = tool_call['function']['arguments'] -%}{%- set ns = namespace(arguments_list=[]) -%}{%- for key, value in args.items() -%}{%- set ns.arguments_list = ns.arguments_list + [key ~ '=' ~ (value | tojson)] -%}{%- endfor -%}{%- set arguments = ns.arguments_list | join(', ') -%}{{- tool_call['function']['name'] + '(' + arguments + ')' -}}{%- if not loop.last -%}{{ '
+' }}{%- endif -%}{% else %}{{- tool_call -}}{%- endif %}{%- endfor %}{{- '</function_calls>' -}}{%- endif -%}{%- if not loop.last -%}{{- '<|im_end|>' + '
+' -}}{%- else -%}{{- eos_token -}}{%- endif -%}{%- elif message['role'] == 'environment' -%}{{- '<|im_start|>environment
+' + message['content'] + '<|im_end|>
+' -}}{%- elif message['role'] == 'tool' -%}{{- '<|im_start|>environment
+' + message['content'] + '<|im_end|>
+' -}}{%- endif -%}{%- if loop.last and add_generation_prompt -%}{{- '<|im_start|>assistant
+' -}}{%- endif -%}{%- endfor -%}

config.json

@@ -0,0 +1,75 @@
+{
+  "architectures": [
+    "Olmo3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 65536,
+  "model_type": "olmo3",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "full_attention": {
+      "attention_factor": 1.2079441541679836,
+      "beta_fast": 32,
+      "beta_slow": 1,
+      "factor": 8.0,
+      "original_max_position_embeddings": 8192,
+      "rope_theta": 500000,
+      "rope_type": "yarn"
+    },
+    "sliding_attention": {
+      "rope_theta": 500000.0,
+      "rope_type": "default"
+    }
+  },
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.17.0",
+  "use_cache": true,
+  "vocab_size": 100278
+}

generation_config.json

@@ -0,0 +1,3 @@
+{
+  "transformers_version": "5.17.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c5520bc3083df5963cb5de190b2da27241b02e5af6dce656d0e07be7e3a310c
+size 14596063960

run_manifest.json

@@ -0,0 +1,6 @@
+{
+ "arm": "olmo-pretrained",
+ "exported_checkpoint": "checkpoint-300",
+ "export_rule": "Olmo common step 300 (2026-10-16): no step had every arm passing GATE_OLMO; 300 was the most even (TAC 32-39/52 completed for all arms)",
+ "recipe": "legacy part one, RoPE-guarded"
+}

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 65536,
+  "pad_token": "<|pad|>",
+  "padding_side": "right",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

trainer_state.json

@@ -0,0 +1,511 @@
+{
+  "best_global_step": 300,
+  "best_metric": 0.13610543310642242,
+  "best_model_checkpoint": "/workspace/anchor/olmo-pretrained/train/checkpoint-300",
+  "epoch": 0.3482298316889147,
+  "eval_steps": 50,
+  "global_step": 300,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.005803830528148578,
+      "grad_norm": 1.8992259502410889,
+      "learning_rate": 1.8181818181818182e-05,
+      "loss": 0.9941,
+      "step": 5
+    },
+    {
+      "epoch": 0.011607661056297156,
+      "grad_norm": 1.5108100175857544,
+      "learning_rate": 4.0909090909090915e-05,
+      "loss": 0.8727,
+      "step": 10
+    },
+    {
+      "epoch": 0.017411491584445733,
+      "grad_norm": 1.313894271850586,
+      "learning_rate": 6.363636363636364e-05,
+      "loss": 0.7164,
+      "step": 15
+    },
+    {
+      "epoch": 0.023215322112594312,
+      "grad_norm": 1.5354955196380615,
+      "learning_rate": 8.636363636363637e-05,
+      "loss": 0.5324,
+      "step": 20
+    },
+    {
+      "epoch": 0.02901915264074289,
+      "grad_norm": 1.4390828609466553,
+      "learning_rate": 0.00010909090909090909,
+      "loss": 0.4492,
+      "step": 25
+    },
+    {
+      "epoch": 0.034822983168891465,
+      "grad_norm": 1.8266173601150513,
+      "learning_rate": 0.0001318181818181818,
+      "loss": 0.4172,
+      "step": 30
+    },
+    {
+      "epoch": 0.04062681369704005,
+      "grad_norm": 1.5203430652618408,
+      "learning_rate": 0.00015454545454545454,
+      "loss": 0.3966,
+      "step": 35
+    },
+    {
+      "epoch": 0.046430644225188625,
+      "grad_norm": 1.6018481254577637,
+      "learning_rate": 0.00017727272727272728,
+      "loss": 0.3584,
+      "step": 40
+    },
+    {
+      "epoch": 0.0522344747533372,
+      "grad_norm": 1.3058902025222778,
+      "learning_rate": 0.0002,
+      "loss": 0.3571,
+      "step": 45
+    },
+    {
+      "epoch": 0.05803830528148578,
+      "grad_norm": 1.388102650642395,
+      "learning_rate": 0.00019998156301911284,
+      "loss": 0.2975,
+      "step": 50
+    },
+    {
+      "epoch": 0.05803830528148578,
+      "eval_loss": 0.2423536628484726,
+      "eval_runtime": 152.5048,
+      "eval_samples_per_second": 2.321,
+      "eval_steps_per_second": 1.161,
+      "step": 50
+    },
+    {
+      "epoch": 0.06384213580963435,
+      "grad_norm": 1.3211257457733154,
+      "learning_rate": 0.0001999262588748966,
+      "loss": 0.3098,
+      "step": 55
+    },
+    {
+      "epoch": 0.06964596633778293,
+      "grad_norm": 1.0737800598144531,
+      "learning_rate": 0.00019983410796018032,
+      "loss": 0.27,
+      "step": 60
+    },
+    {
+      "epoch": 0.07544979686593152,
+      "grad_norm": 1.062996745109558,
+      "learning_rate": 0.00019970514425465705,
+      "loss": 0.2597,
+      "step": 65
+    },
+    {
+      "epoch": 0.0812536273940801,
+      "grad_norm": 0.8997095227241516,
+      "learning_rate": 0.00019953941531235424,
+      "loss": 0.2213,
+      "step": 70
+    },
+    {
+      "epoch": 0.08705745792222867,
+      "grad_norm": 0.6930145025253296,
+      "learning_rate": 0.00019933698224409876,
+      "loss": 0.2616,
+      "step": 75
+    },
+    {
+      "epoch": 0.09286128845037725,
+      "grad_norm": 0.5634636282920837,
+      "learning_rate": 0.0001990979196949828,
+      "loss": 0.2218,
+      "step": 80
+    },
+    {
+      "epoch": 0.09866511897852583,
+      "grad_norm": 0.545459508895874,
+      "learning_rate": 0.00019882231581683937,
+      "loss": 0.238,
+      "step": 85
+    },
+    {
+      "epoch": 0.1044689495066744,
+      "grad_norm": 0.4912775456905365,
+      "learning_rate": 0.0001985102722357371,
+      "loss": 0.2145,
+      "step": 90
+    },
+    {
+      "epoch": 0.11027278003482298,
+      "grad_norm": 0.4108012914657593,
+      "learning_rate": 0.0001981619040145068,
+      "loss": 0.2336,
+      "step": 95
+    },
+    {
+      "epoch": 0.11607661056297155,
+      "grad_norm": 0.521858274936676,
+      "learning_rate": 0.00019777733961031326,
+      "loss": 0.1963,
+      "step": 100
+    },
+    {
+      "epoch": 0.11607661056297155,
+      "eval_loss": 0.1702147275209427,
+      "eval_runtime": 57.6184,
+      "eval_samples_per_second": 6.144,
+      "eval_steps_per_second": 3.072,
+      "step": 100
+    },
+    {
+      "epoch": 0.12188044109112015,
+      "grad_norm": 0.5323580503463745,
+      "learning_rate": 0.00019735672082728782,
+      "loss": 0.2433,
+      "step": 105
+    },
+    {
+      "epoch": 0.1276842716192687,
+      "grad_norm": 0.5639608502388,
+      "learning_rate": 0.0001969002027642398,
+      "loss": 0.2063,
+      "step": 110
+    },
+    {
+      "epoch": 0.1334881021474173,
+      "grad_norm": 0.583233654499054,
+      "learning_rate": 0.00019640795375746518,
+      "loss": 0.2208,
+      "step": 115
+    },
+    {
+      "epoch": 0.13929193267556586,
+      "grad_norm": 0.5018272399902344,
+      "learning_rate": 0.00019588015531867464,
+      "loss": 0.204,
+      "step": 120
+    },
+    {
+      "epoch": 0.14509576320371445,
+      "grad_norm": 0.48238405585289,
+      "learning_rate": 0.00019531700206806274,
+      "loss": 0.2155,
+      "step": 125
+    },
+    {
+      "epoch": 0.15089959373186304,
+      "grad_norm": 0.4539037048816681,
+      "learning_rate": 0.00019471870166254377,
+      "loss": 0.208,
+      "step": 130
+    },
+    {
+      "epoch": 0.1567034242600116,
+      "grad_norm": 0.5225255489349365,
+      "learning_rate": 0.00019408547471918061,
+      "loss": 0.2292,
+      "step": 135
+    },
+    {
+      "epoch": 0.1625072547881602,
+      "grad_norm": 0.4645092785358429,
+      "learning_rate": 0.00019341755473383432,
+      "loss": 0.2074,
+      "step": 140
+    },
+    {
+      "epoch": 0.16831108531630876,
+      "grad_norm": 0.47805657982826233,
+      "learning_rate": 0.00019271518799506492,
+      "loss": 0.2301,
+      "step": 145
+    },
+    {
+      "epoch": 0.17411491584445735,
+      "grad_norm": 0.43033567070961,
+      "learning_rate": 0.00019197863349331522,
+      "loss": 0.1916,
+      "step": 150
+    },
+    {
+      "epoch": 0.17411491584445735,
+      "eval_loss": 0.1569332629442215,
+      "eval_runtime": 57.9274,
+      "eval_samples_per_second": 6.111,
+      "eval_steps_per_second": 3.056,
+      "step": 150
+    },
+    {
+      "epoch": 0.1799187463726059,
+      "grad_norm": 0.5715239644050598,
+      "learning_rate": 0.00019120816282541063,
+      "loss": 0.2054,
+      "step": 155
+    },
+    {
+      "epoch": 0.1857225769007545,
+      "grad_norm": 0.5529009699821472,
+      "learning_rate": 0.00019040406009441073,
+      "loss": 0.2156,
+      "step": 160
+    },
+    {
+      "epoch": 0.1915264074289031,
+      "grad_norm": 0.5217124819755554,
+      "learning_rate": 0.00018956662180484913,
+      "loss": 0.227,
+      "step": 165
+    },
+    {
+      "epoch": 0.19733023795705165,
+      "grad_norm": 0.4713853895664215,
+      "learning_rate": 0.00018869615675340068,
+      "loss": 0.2191,
+      "step": 170
+    },
+    {
+      "epoch": 0.20313406848520024,
+      "grad_norm": 0.44989559054374695,
+      "learning_rate": 0.00018779298591501564,
+      "loss": 0.2052,
+      "step": 175
+    },
+    {
+      "epoch": 0.2089378990133488,
+      "grad_norm": 0.5454024076461792,
+      "learning_rate": 0.00018685744232456374,
+      "loss": 0.2009,
+      "step": 180
+    },
+    {
+      "epoch": 0.2147417295414974,
+      "grad_norm": 0.5243939161300659,
+      "learning_rate": 0.0001858898709540309,
+      "loss": 0.175,
+      "step": 185
+    },
+    {
+      "epoch": 0.22054556006964596,
+      "grad_norm": 0.4877641499042511,
+      "learning_rate": 0.0001848906285853145,
+      "loss": 0.1861,
+      "step": 190
+    },
+    {
+      "epoch": 0.22634939059779455,
+      "grad_norm": 0.45391249656677246,
+      "learning_rate": 0.0001838600836786635,
+      "loss": 0.1775,
+      "step": 195
+    },
+    {
+      "epoch": 0.2321532211259431,
+      "grad_norm": 0.49695494771003723,
+      "learning_rate": 0.0001827986162368129,
+      "loss": 0.1791,
+      "step": 200
+    },
+    {
+      "epoch": 0.2321532211259431,
+      "eval_loss": 0.14798370003700256,
+      "eval_runtime": 57.9291,
+      "eval_samples_per_second": 6.111,
+      "eval_steps_per_second": 3.055,
+      "step": 200
+    },
+    {
+      "epoch": 0.2379570516540917,
+      "grad_norm": 0.3668426275253296,
+      "learning_rate": 0.0001817066176648615,
+      "loss": 0.2095,
+      "step": 205
+    },
+    {
+      "epoch": 0.2437608821822403,
+      "grad_norm": 0.48783811926841736,
+      "learning_rate": 0.0001805844906259452,
+      "loss": 0.2096,
+      "step": 210
+    },
+    {
+      "epoch": 0.24956471271038885,
+      "grad_norm": 0.3652338981628418,
+      "learning_rate": 0.00017943264889275944,
+      "loss": 0.1579,
+      "step": 215
+    },
+    {
+      "epoch": 0.2553685432385374,
+      "grad_norm": 0.4943305552005768,
+      "learning_rate": 0.00017825151719498466,
+      "loss": 0.169,
+      "step": 220
+    },
+    {
+      "epoch": 0.261172373766686,
+      "grad_norm": 0.4297771453857422,
+      "learning_rate": 0.0001770415310626715,
+      "loss": 0.1633,
+      "step": 225
+    },
+    {
+      "epoch": 0.2669762042948346,
+      "grad_norm": 0.5426889657974243,
+      "learning_rate": 0.00017580313666564395,
+      "loss": 0.1713,
+      "step": 230
+    },
+    {
+      "epoch": 0.2727800348229832,
+      "grad_norm": 0.5065458416938782,
+      "learning_rate": 0.0001745367906489786,
+      "loss": 0.1921,
+      "step": 235
+    },
+    {
+      "epoch": 0.2785838653511317,
+      "grad_norm": 0.452774316072464,
+      "learning_rate": 0.00017324295996462146,
+      "loss": 0.2107,
+      "step": 240
+    },
+    {
+      "epoch": 0.2843876958792803,
+      "grad_norm": 0.37844130396842957,
+      "learning_rate": 0.00017192212169920459,
+      "loss": 0.1468,
+      "step": 245
+    },
+    {
+      "epoch": 0.2901915264074289,
+      "grad_norm": 0.3819119930267334,
+      "learning_rate": 0.00017057476289812504,
+      "loss": 0.1421,
+      "step": 250
+    },
+    {
+      "epoch": 0.2901915264074289,
+      "eval_loss": 0.1410292237997055,
+      "eval_runtime": 57.9112,
+      "eval_samples_per_second": 6.113,
+      "eval_steps_per_second": 3.056,
+      "step": 250
+    },
+    {
+      "epoch": 0.2959953569355775,
+      "grad_norm": 0.41748952865600586,
+      "learning_rate": 0.00016920138038595216,
+      "loss": 0.1774,
+      "step": 255
+    },
+    {
+      "epoch": 0.3017991874637261,
+      "grad_norm": 0.4169933795928955,
+      "learning_rate": 0.0001678024805832284,
+      "loss": 0.207,
+      "step": 260
+    },
+    {
+      "epoch": 0.3076030179918746,
+      "grad_norm": 0.3983752727508545,
+      "learning_rate": 0.0001663785793197323,
+      "loss": 0.1492,
+      "step": 265
+    },
+    {
+      "epoch": 0.3134068485200232,
+      "grad_norm": 0.3772595524787903,
+      "learning_rate": 0.00016493020164427152,
+      "loss": 0.1154,
+      "step": 270
+    },
+    {
+      "epoch": 0.3192106790481718,
+      "grad_norm": 0.3978261947631836,
+      "learning_rate": 0.00016345788163107646,
+      "loss": 0.177,
+      "step": 275
+    },
+    {
+      "epoch": 0.3250145095763204,
+      "grad_norm": 0.41389644145965576,
+      "learning_rate": 0.0001619621621828659,
+      "loss": 0.2136,
+      "step": 280
+    },
+    {
+      "epoch": 0.3308183401044689,
+      "grad_norm": 0.42459338903427124,
+      "learning_rate": 0.0001604435948306575,
+      "loss": 0.1915,
+      "step": 285
+    },
+    {
+      "epoch": 0.3366221706326175,
+      "grad_norm": 0.41835540533065796,
+      "learning_rate": 0.0001589027395303962,
+      "loss": 0.1747,
+      "step": 290
+    },
+    {
+      "epoch": 0.3424260011607661,
+      "grad_norm": 0.5012316703796387,
+      "learning_rate": 0.0001573401644564764,
+      "loss": 0.1698,
+      "step": 295
+    },
+    {
+      "epoch": 0.3482298316889147,
+      "grad_norm": 0.43907007575035095,
+      "learning_rate": 0.00015575644579223362,
+      "loss": 0.1693,
+      "step": 300
+    },
+    {
+      "epoch": 0.3482298316889147,
+      "eval_loss": 0.13610543310642242,
+      "eval_runtime": 57.7605,
+      "eval_samples_per_second": 6.129,
+      "eval_steps_per_second": 3.064,
+      "step": 300
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 862,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 2,
+        "early_stopping_threshold": 0.001
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8.372265212159263e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}