Upload continuous evaluation checkpoints, batch 27

continuous/smollm2/clip_openai__l14/finetune/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25991c50d1a5642b136759ce9ab3460a1afd5b544366cef5c00c93ecbf92db41
+size 4255708192

continuous/smollm2/clip_openai__l14/finetune/special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

continuous/smollm2/clip_openai__l14/finetune/tokenizer_config.json

@@ -0,0 +1,155 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

continuous/smollm2/clip_openai__l14/finetune/trainer_state.json

@@ -0,0 +1,1856 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 2597,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.003850596842510589,
+      "grad_norm": 3.2972614765167236,
+      "learning_rate": 1.153846153846154e-06,
+      "loss": 1.2529,
+      "step": 10
+    },
+    {
+      "epoch": 0.007701193685021178,
+      "grad_norm": 2.0281827449798584,
+      "learning_rate": 2.435897435897436e-06,
+      "loss": 1.1079,
+      "step": 20
+    },
+    {
+      "epoch": 0.011551790527531768,
+      "grad_norm": 1.9253156185150146,
+      "learning_rate": 3.7179487179487184e-06,
+      "loss": 0.9886,
+      "step": 30
+    },
+    {
+      "epoch": 0.015402387370042356,
+      "grad_norm": 1.3174968957901,
+      "learning_rate": 5e-06,
+      "loss": 0.9442,
+      "step": 40
+    },
+    {
+      "epoch": 0.019252984212552945,
+      "grad_norm": 1.289121150970459,
+      "learning_rate": 6.282051282051282e-06,
+      "loss": 0.9332,
+      "step": 50
+    },
+    {
+      "epoch": 0.023103581055063535,
+      "grad_norm": 0.28699904680252075,
+      "learning_rate": 7.564102564102564e-06,
+      "loss": 0.8647,
+      "step": 60
+    },
+    {
+      "epoch": 0.026954177897574125,
+      "grad_norm": 0.22930826246738434,
+      "learning_rate": 8.846153846153847e-06,
+      "loss": 0.8198,
+      "step": 70
+    },
+    {
+      "epoch": 0.030804774740084712,
+      "grad_norm": 0.23816744983196259,
+      "learning_rate": 9.999996111488723e-06,
+      "loss": 0.8331,
+      "step": 80
+    },
+    {
+      "epoch": 0.0346553715825953,
+      "grad_norm": 0.23513822257518768,
+      "learning_rate": 9.999529497453782e-06,
+      "loss": 0.8183,
+      "step": 90
+    },
+    {
+      "epoch": 0.03850596842510589,
+      "grad_norm": 0.19711492955684662,
+      "learning_rate": 9.998285264324193e-06,
+      "loss": 0.7872,
+      "step": 100
+    },
+    {
+      "epoch": 0.04235656526761648,
+      "grad_norm": 0.21673457324504852,
+      "learning_rate": 9.996263605626052e-06,
+      "loss": 0.7804,
+      "step": 110
+    },
+    {
+      "epoch": 0.04620716211012707,
+      "grad_norm": 0.2207483947277069,
+      "learning_rate": 9.99346483580503e-06,
+      "loss": 0.7995,
+      "step": 120
+    },
+    {
+      "epoch": 0.05005775895263766,
+      "grad_norm": 0.2070060819387436,
+      "learning_rate": 9.989889390177467e-06,
+      "loss": 0.7733,
+      "step": 130
+    },
+    {
+      "epoch": 0.05390835579514825,
+      "grad_norm": 0.22163434326648712,
+      "learning_rate": 9.985537824862647e-06,
+      "loss": 0.7896,
+      "step": 140
+    },
+    {
+      "epoch": 0.05775895263765884,
+      "grad_norm": 0.2299714833498001,
+      "learning_rate": 9.980410816696317e-06,
+      "loss": 0.7668,
+      "step": 150
+    },
+    {
+      "epoch": 0.061609549480169425,
+      "grad_norm": 0.21941758692264557,
+      "learning_rate": 9.97450916312541e-06,
+      "loss": 0.7735,
+      "step": 160
+    },
+    {
+      "epoch": 0.06546014632268002,
+      "grad_norm": 0.19399426877498627,
+      "learning_rate": 9.967833782084004e-06,
+      "loss": 0.7733,
+      "step": 170
+    },
+    {
+      "epoch": 0.0693107431651906,
+      "grad_norm": 0.20675109326839447,
+      "learning_rate": 9.960385711850554e-06,
+      "loss": 0.8043,
+      "step": 180
+    },
+    {
+      "epoch": 0.07316134000770119,
+      "grad_norm": 0.20418842136859894,
+      "learning_rate": 9.952166110886394e-06,
+      "loss": 0.7924,
+      "step": 190
+    },
+    {
+      "epoch": 0.07701193685021178,
+      "grad_norm": 0.19385431706905365,
+      "learning_rate": 9.943176257655567e-06,
+      "loss": 0.7891,
+      "step": 200
+    },
+    {
+      "epoch": 0.08086253369272237,
+      "grad_norm": 0.2126033753156662,
+      "learning_rate": 9.933417550425952e-06,
+      "loss": 0.7793,
+      "step": 210
+    },
+    {
+      "epoch": 0.08471313053523297,
+      "grad_norm": 0.1886177510023117,
+      "learning_rate": 9.9228915070518e-06,
+      "loss": 0.7669,
+      "step": 220
+    },
+    {
+      "epoch": 0.08856372737774355,
+      "grad_norm": 0.19311988353729248,
+      "learning_rate": 9.911599764737632e-06,
+      "loss": 0.7634,
+      "step": 230
+    },
+    {
+      "epoch": 0.09241432422025414,
+      "grad_norm": 0.19199371337890625,
+      "learning_rate": 9.899544079783606e-06,
+      "loss": 0.7579,
+      "step": 240
+    },
+    {
+      "epoch": 0.09626492106276473,
+      "grad_norm": 0.19620512425899506,
+      "learning_rate": 9.886726327312338e-06,
+      "loss": 0.7757,
+      "step": 250
+    },
+    {
+      "epoch": 0.10011551790527531,
+      "grad_norm": 0.1766020506620407,
+      "learning_rate": 9.87314850097724e-06,
+      "loss": 0.7717,
+      "step": 260
+    },
+    {
+      "epoch": 0.1039661147477859,
+      "grad_norm": 0.18801046907901764,
+      "learning_rate": 9.858812712652452e-06,
+      "loss": 0.7569,
+      "step": 270
+    },
+    {
+      "epoch": 0.1078167115902965,
+      "grad_norm": 0.18290388584136963,
+      "learning_rate": 9.843721192104341e-06,
+      "loss": 0.7526,
+      "step": 280
+    },
+    {
+      "epoch": 0.11166730843280709,
+      "grad_norm": 0.19425159692764282,
+      "learning_rate": 9.827876286644698e-06,
+      "loss": 0.7855,
+      "step": 290
+    },
+    {
+      "epoch": 0.11551790527531768,
+      "grad_norm": 0.17539188265800476,
+      "learning_rate": 9.811280460765643e-06,
+      "loss": 0.7441,
+      "step": 300
+    },
+    {
+      "epoch": 0.11936850211782826,
+      "grad_norm": 0.18481691181659698,
+      "learning_rate": 9.793936295756292e-06,
+      "loss": 0.7486,
+      "step": 310
+    },
+    {
+      "epoch": 0.12321909896033885,
+      "grad_norm": 0.1922149360179901,
+      "learning_rate": 9.775846489301279e-06,
+      "loss": 0.7571,
+      "step": 320
+    },
+    {
+      "epoch": 0.12706969580284944,
+      "grad_norm": 0.18815866112709045,
+      "learning_rate": 9.757013855061157e-06,
+      "loss": 0.7588,
+      "step": 330
+    },
+    {
+      "epoch": 0.13092029264536004,
+      "grad_norm": 0.19353695213794708,
+      "learning_rate": 9.737441322234758e-06,
+      "loss": 0.7629,
+      "step": 340
+    },
+    {
+      "epoch": 0.1347708894878706,
+      "grad_norm": 0.2056358903646469,
+      "learning_rate": 9.717131935103605e-06,
+      "loss": 0.7688,
+      "step": 350
+    },
+    {
+      "epoch": 0.1386214863303812,
+      "grad_norm": 0.18176105618476868,
+      "learning_rate": 9.696088852558398e-06,
+      "loss": 0.75,
+      "step": 360
+    },
+    {
+      "epoch": 0.1424720831728918,
+      "grad_norm": 0.2042083591222763,
+      "learning_rate": 9.674315347607686e-06,
+      "loss": 0.7788,
+      "step": 370
+    },
+    {
+      "epoch": 0.14632268001540238,
+      "grad_norm": 0.19040749967098236,
+      "learning_rate": 9.651814806868796e-06,
+      "loss": 0.7437,
+      "step": 380
+    },
+    {
+      "epoch": 0.15017327685791299,
+      "grad_norm": 0.1836700737476349,
+      "learning_rate": 9.62859073004107e-06,
+      "loss": 0.7452,
+      "step": 390
+    },
+    {
+      "epoch": 0.15402387370042356,
+      "grad_norm": 0.17053930461406708,
+      "learning_rate": 9.604646729361547e-06,
+      "loss": 0.7518,
+      "step": 400
+    },
+    {
+      "epoch": 0.15787447054293416,
+      "grad_norm": 0.19875766336917877,
+      "learning_rate": 9.579986529043099e-06,
+      "loss": 0.7486,
+      "step": 410
+    },
+    {
+      "epoch": 0.16172506738544473,
+      "grad_norm": 0.19116470217704773,
+      "learning_rate": 9.554613964695189e-06,
+      "loss": 0.7649,
+      "step": 420
+    },
+    {
+      "epoch": 0.16557566422795533,
+      "grad_norm": 0.18611662089824677,
+      "learning_rate": 9.528532982727281e-06,
+      "loss": 0.7466,
+      "step": 430
+    },
+    {
+      "epoch": 0.16942626107046593,
+      "grad_norm": 0.18092520534992218,
+      "learning_rate": 9.501747639735018e-06,
+      "loss": 0.7526,
+      "step": 440
+    },
+    {
+      "epoch": 0.1732768579129765,
+      "grad_norm": 0.19146224856376648,
+      "learning_rate": 9.474262101869272e-06,
+      "loss": 0.7383,
+      "step": 450
+    },
+    {
+      "epoch": 0.1771274547554871,
+      "grad_norm": 0.2039482742547989,
+      "learning_rate": 9.446080644188141e-06,
+      "loss": 0.7477,
+      "step": 460
+    },
+    {
+      "epoch": 0.18097805159799768,
+      "grad_norm": 0.169089674949646,
+      "learning_rate": 9.417207649992017e-06,
+      "loss": 0.7494,
+      "step": 470
+    },
+    {
+      "epoch": 0.18482864844050828,
+      "grad_norm": 0.19739316403865814,
+      "learning_rate": 9.387647610141816e-06,
+      "loss": 0.7468,
+      "step": 480
+    },
+    {
+      "epoch": 0.18867924528301888,
+      "grad_norm": 0.18184153735637665,
+      "learning_rate": 9.35740512236047e-06,
+      "loss": 0.7424,
+      "step": 490
+    },
+    {
+      "epoch": 0.19252984212552945,
+      "grad_norm": 0.18619920313358307,
+      "learning_rate": 9.326484890517809e-06,
+      "loss": 0.7467,
+      "step": 500
+    },
+    {
+      "epoch": 0.19638043896804006,
+      "grad_norm": 0.1945374459028244,
+      "learning_rate": 9.294891723898926e-06,
+      "loss": 0.7357,
+      "step": 510
+    },
+    {
+      "epoch": 0.20023103581055063,
+      "grad_norm": 0.18352735042572021,
+      "learning_rate": 9.262630536456153e-06,
+      "loss": 0.7594,
+      "step": 520
+    },
+    {
+      "epoch": 0.20408163265306123,
+      "grad_norm": 0.1937621533870697,
+      "learning_rate": 9.229706346044749e-06,
+      "loss": 0.7419,
+      "step": 530
+    },
+    {
+      "epoch": 0.2079322294955718,
+      "grad_norm": 0.18922854959964752,
+      "learning_rate": 9.196124273642422e-06,
+      "loss": 0.7529,
+      "step": 540
+    },
+    {
+      "epoch": 0.2117828263380824,
+      "grad_norm": 0.18072281777858734,
+      "learning_rate": 9.161889542552842e-06,
+      "loss": 0.7594,
+      "step": 550
+    },
+    {
+      "epoch": 0.215633423180593,
+      "grad_norm": 0.1792808622121811,
+      "learning_rate": 9.127007477593193e-06,
+      "loss": 0.7606,
+      "step": 560
+    },
+    {
+      "epoch": 0.21948402002310358,
+      "grad_norm": 0.18946735560894012,
+      "learning_rate": 9.091483504265974e-06,
+      "loss": 0.7734,
+      "step": 570
+    },
+    {
+      "epoch": 0.22333461686561418,
+      "grad_norm": 0.19394512474536896,
+      "learning_rate": 9.055323147915116e-06,
+      "loss": 0.742,
+      "step": 580
+    },
+    {
+      "epoch": 0.22718521370812475,
+      "grad_norm": 0.18536533415317535,
+      "learning_rate": 9.018532032866585e-06,
+      "loss": 0.7577,
+      "step": 590
+    },
+    {
+      "epoch": 0.23103581055063535,
+      "grad_norm": 0.16979281604290009,
+      "learning_rate": 8.981115881553579e-06,
+      "loss": 0.7543,
+      "step": 600
+    },
+    {
+      "epoch": 0.23488640739314592,
+      "grad_norm": 0.18990707397460938,
+      "learning_rate": 8.943080513626476e-06,
+      "loss": 0.7739,
+      "step": 610
+    },
+    {
+      "epoch": 0.23873700423565652,
+      "grad_norm": 0.1733860969543457,
+      "learning_rate": 8.904431845047641e-06,
+      "loss": 0.7522,
+      "step": 620
+    },
+    {
+      "epoch": 0.24258760107816713,
+      "grad_norm": 0.18061257898807526,
+      "learning_rate": 8.86517588717128e-06,
+      "loss": 0.7643,
+      "step": 630
+    },
+    {
+      "epoch": 0.2464381979206777,
+      "grad_norm": 0.1795412003993988,
+      "learning_rate": 8.82531874580844e-06,
+      "loss": 0.7631,
+      "step": 640
+    },
+    {
+      "epoch": 0.25028879476318827,
+      "grad_norm": 0.16028080880641937,
+      "learning_rate": 8.784866620277314e-06,
+      "loss": 0.723,
+      "step": 650
+    },
+    {
+      "epoch": 0.25413939160569887,
+      "grad_norm": 0.18285559117794037,
+      "learning_rate": 8.743825802439018e-06,
+      "loss": 0.7475,
+      "step": 660
+    },
+    {
+      "epoch": 0.2579899884482095,
+      "grad_norm": 0.17592895030975342,
+      "learning_rate": 8.702202675718956e-06,
+      "loss": 0.7596,
+      "step": 670
+    },
+    {
+      "epoch": 0.2618405852907201,
+      "grad_norm": 0.1666291356086731,
+      "learning_rate": 8.660003714113959e-06,
+      "loss": 0.7383,
+      "step": 680
+    },
+    {
+      "epoch": 0.2656911821332307,
+      "grad_norm": 0.17878305912017822,
+      "learning_rate": 8.617235481185323e-06,
+      "loss": 0.7483,
+      "step": 690
+    },
+    {
+      "epoch": 0.2695417789757412,
+      "grad_norm": 0.24169033765792847,
+      "learning_rate": 8.573904629037931e-06,
+      "loss": 0.762,
+      "step": 700
+    },
+    {
+      "epoch": 0.2733923758182518,
+      "grad_norm": 0.1698446124792099,
+      "learning_rate": 8.530017897285585e-06,
+      "loss": 0.7432,
+      "step": 710
+    },
+    {
+      "epoch": 0.2772429726607624,
+      "grad_norm": 0.17150413990020752,
+      "learning_rate": 8.485582112002746e-06,
+      "loss": 0.7443,
+      "step": 720
+    },
+    {
+      "epoch": 0.281093569503273,
+      "grad_norm": 0.18670643866062164,
+      "learning_rate": 8.440604184662811e-06,
+      "loss": 0.7536,
+      "step": 730
+    },
+    {
+      "epoch": 0.2849441663457836,
+      "grad_norm": 0.17963248491287231,
+      "learning_rate": 8.395091111063118e-06,
+      "loss": 0.7383,
+      "step": 740
+    },
+    {
+      "epoch": 0.28879476318829417,
+      "grad_norm": 0.16416804492473602,
+      "learning_rate": 8.349049970236822e-06,
+      "loss": 0.7365,
+      "step": 750
+    },
+    {
+      "epoch": 0.29264536003080477,
+      "grad_norm": 0.19067895412445068,
+      "learning_rate": 8.302487923351842e-06,
+      "loss": 0.7327,
+      "step": 760
+    },
+    {
+      "epoch": 0.29649595687331537,
+      "grad_norm": 0.16189923882484436,
+      "learning_rate": 8.255412212597021e-06,
+      "loss": 0.7505,
+      "step": 770
+    },
+    {
+      "epoch": 0.30034655371582597,
+      "grad_norm": 0.18577417731285095,
+      "learning_rate": 8.207830160055688e-06,
+      "loss": 0.747,
+      "step": 780
+    },
+    {
+      "epoch": 0.3041971505583365,
+      "grad_norm": 0.17445193231105804,
+      "learning_rate": 8.159749166566786e-06,
+      "loss": 0.7291,
+      "step": 790
+    },
+    {
+      "epoch": 0.3080477474008471,
+      "grad_norm": 0.1783832311630249,
+      "learning_rate": 8.11117671057377e-06,
+      "loss": 0.7519,
+      "step": 800
+    },
+    {
+      "epoch": 0.3118983442433577,
+      "grad_norm": 0.18522772192955017,
+      "learning_rate": 8.062120346961404e-06,
+      "loss": 0.7503,
+      "step": 810
+    },
+    {
+      "epoch": 0.3157489410858683,
+      "grad_norm": 0.17171914875507355,
+      "learning_rate": 8.012587705880705e-06,
+      "loss": 0.7471,
+      "step": 820
+    },
+    {
+      "epoch": 0.3195995379283789,
+      "grad_norm": 0.1816757470369339,
+      "learning_rate": 7.962586491562146e-06,
+      "loss": 0.7328,
+      "step": 830
+    },
+    {
+      "epoch": 0.32345013477088946,
+      "grad_norm": 0.18051651120185852,
+      "learning_rate": 7.912124481117357e-06,
+      "loss": 0.7344,
+      "step": 840
+    },
+    {
+      "epoch": 0.32730073161340006,
+      "grad_norm": 0.16732724010944366,
+      "learning_rate": 7.861209523329494e-06,
+      "loss": 0.7439,
+      "step": 850
+    },
+    {
+      "epoch": 0.33115132845591067,
+      "grad_norm": 0.18085485696792603,
+      "learning_rate": 7.809849537432432e-06,
+      "loss": 0.734,
+      "step": 860
+    },
+    {
+      "epoch": 0.33500192529842127,
+      "grad_norm": 0.1737603396177292,
+      "learning_rate": 7.758052511879035e-06,
+      "loss": 0.7377,
+      "step": 870
+    },
+    {
+      "epoch": 0.33885252214093187,
+      "grad_norm": 0.1958928108215332,
+      "learning_rate": 7.705826503098642e-06,
+      "loss": 0.7429,
+      "step": 880
+    },
+    {
+      "epoch": 0.3427031189834424,
+      "grad_norm": 0.17915913462638855,
+      "learning_rate": 7.653179634243979e-06,
+      "loss": 0.7351,
+      "step": 890
+    },
+    {
+      "epoch": 0.346553715825953,
+      "grad_norm": 0.17541474103927612,
+      "learning_rate": 7.600120093927696e-06,
+      "loss": 0.737,
+      "step": 900
+    },
+    {
+      "epoch": 0.3504043126684636,
+      "grad_norm": 0.2014024406671524,
+      "learning_rate": 7.546656134948729e-06,
+      "loss": 0.7264,
+      "step": 910
+    },
+    {
+      "epoch": 0.3542549095109742,
+      "grad_norm": 0.16445063054561615,
+      "learning_rate": 7.492796073008661e-06,
+      "loss": 0.7358,
+      "step": 920
+    },
+    {
+      "epoch": 0.3581055063534848,
+      "grad_norm": 0.19084401428699493,
+      "learning_rate": 7.438548285418326e-06,
+      "loss": 0.7174,
+      "step": 930
+    },
+    {
+      "epoch": 0.36195610319599536,
+      "grad_norm": 0.18842382729053497,
+      "learning_rate": 7.3839212097948e-06,
+      "loss": 0.7487,
+      "step": 940
+    },
+    {
+      "epoch": 0.36580670003850596,
+      "grad_norm": 0.18046307563781738,
+      "learning_rate": 7.328923342749039e-06,
+      "loss": 0.7639,
+      "step": 950
+    },
+    {
+      "epoch": 0.36965729688101656,
+      "grad_norm": 0.17019270360469818,
+      "learning_rate": 7.273563238564319e-06,
+      "loss": 0.7598,
+      "step": 960
+    },
+    {
+      "epoch": 0.37350789372352716,
+      "grad_norm": 0.17679363489151,
+      "learning_rate": 7.217849507865724e-06,
+      "loss": 0.7281,
+      "step": 970
+    },
+    {
+      "epoch": 0.37735849056603776,
+      "grad_norm": 0.16904659569263458,
+      "learning_rate": 7.161790816280858e-06,
+      "loss": 0.7384,
+      "step": 980
+    },
+    {
+      "epoch": 0.3812090874085483,
+      "grad_norm": 0.18779674172401428,
+      "learning_rate": 7.1053958830920125e-06,
+      "loss": 0.724,
+      "step": 990
+    },
+    {
+      "epoch": 0.3850596842510589,
+      "grad_norm": 0.1871771663427353,
+      "learning_rate": 7.0486734798799715e-06,
+      "loss": 0.7386,
+      "step": 1000
+    },
+    {
+      "epoch": 0.3889102810935695,
+      "grad_norm": 0.1688663363456726,
+      "learning_rate": 6.991632429159708e-06,
+      "loss": 0.7563,
+      "step": 1010
+    },
+    {
+      "epoch": 0.3927608779360801,
+      "grad_norm": 0.19467109441757202,
+      "learning_rate": 6.934281603008132e-06,
+      "loss": 0.7544,
+      "step": 1020
+    },
+    {
+      "epoch": 0.39661147477859066,
+      "grad_norm": 0.15688785910606384,
+      "learning_rate": 6.876629921684143e-06,
+      "loss": 0.7507,
+      "step": 1030
+    },
+    {
+      "epoch": 0.40046207162110126,
+      "grad_norm": 0.19135324656963348,
+      "learning_rate": 6.818686352241199e-06,
+      "loss": 0.7387,
+      "step": 1040
+    },
+    {
+      "epoch": 0.40431266846361186,
+      "grad_norm": 0.16503691673278809,
+      "learning_rate": 6.760459907132573e-06,
+      "loss": 0.7334,
+      "step": 1050
+    },
+    {
+      "epoch": 0.40816326530612246,
+      "grad_norm": 0.17050068080425262,
+      "learning_rate": 6.701959642809589e-06,
+      "loss": 0.7398,
+      "step": 1060
+    },
+    {
+      "epoch": 0.41201386214863306,
+      "grad_norm": 0.17295043170452118,
+      "learning_rate": 6.643194658312987e-06,
+      "loss": 0.7311,
+      "step": 1070
+    },
+    {
+      "epoch": 0.4158644589911436,
+      "grad_norm": 0.15935075283050537,
+      "learning_rate": 6.584174093857676e-06,
+      "loss": 0.7564,
+      "step": 1080
+    },
+    {
+      "epoch": 0.4197150558336542,
+      "grad_norm": 0.18899163603782654,
+      "learning_rate": 6.5249071294110735e-06,
+      "loss": 0.7402,
+      "step": 1090
+    },
+    {
+      "epoch": 0.4235656526761648,
+      "grad_norm": 0.18495947122573853,
+      "learning_rate": 6.465402983265274e-06,
+      "loss": 0.7519,
+      "step": 1100
+    },
+    {
+      "epoch": 0.4274162495186754,
+      "grad_norm": 0.17934852838516235,
+      "learning_rate": 6.405670910603243e-06,
+      "loss": 0.7439,
+      "step": 1110
+    },
+    {
+      "epoch": 0.431266846361186,
+      "grad_norm": 0.1835138201713562,
+      "learning_rate": 6.345720202059291e-06,
+      "loss": 0.7414,
+      "step": 1120
+    },
+    {
+      "epoch": 0.43511744320369655,
+      "grad_norm": 0.17690949141979218,
+      "learning_rate": 6.285560182274008e-06,
+      "loss": 0.7222,
+      "step": 1130
+    },
+    {
+      "epoch": 0.43896804004620715,
+      "grad_norm": 0.1865280568599701,
+      "learning_rate": 6.225200208443936e-06,
+      "loss": 0.7467,
+      "step": 1140
+    },
+    {
+      "epoch": 0.44281863688871775,
+      "grad_norm": 0.17375321686267853,
+      "learning_rate": 6.16464966886616e-06,
+      "loss": 0.7305,
+      "step": 1150
+    },
+    {
+      "epoch": 0.44666923373122835,
+      "grad_norm": 0.20874516665935516,
+      "learning_rate": 6.103917981478065e-06,
+      "loss": 0.761,
+      "step": 1160
+    },
+    {
+      "epoch": 0.45051983057373896,
+      "grad_norm": 0.1714327484369278,
+      "learning_rate": 6.043014592392489e-06,
+      "loss": 0.7672,
+      "step": 1170
+    },
+    {
+      "epoch": 0.4543704274162495,
+      "grad_norm": 0.1761511266231537,
+      "learning_rate": 5.981948974428485e-06,
+      "loss": 0.7483,
+      "step": 1180
+    },
+    {
+      "epoch": 0.4582210242587601,
+      "grad_norm": 0.17268823087215424,
+      "learning_rate": 5.920730625637934e-06,
+      "loss": 0.723,
+      "step": 1190
+    },
+    {
+      "epoch": 0.4620716211012707,
+      "grad_norm": 0.18802501261234283,
+      "learning_rate": 5.8593690678282355e-06,
+      "loss": 0.7327,
+      "step": 1200
+    },
+    {
+      "epoch": 0.4659222179437813,
+      "grad_norm": 0.17421527206897736,
+      "learning_rate": 5.797873845081301e-06,
+      "loss": 0.7312,
+      "step": 1210
+    },
+    {
+      "epoch": 0.46977281478629185,
+      "grad_norm": 0.17880789935588837,
+      "learning_rate": 5.736254522269075e-06,
+      "loss": 0.7565,
+      "step": 1220
+    },
+    {
+      "epoch": 0.47362341162880245,
+      "grad_norm": 0.16892190277576447,
+      "learning_rate": 5.6745206835658386e-06,
+      "loss": 0.7529,
+      "step": 1230
+    },
+    {
+      "epoch": 0.47747400847131305,
+      "grad_norm": 0.1595633625984192,
+      "learning_rate": 5.612681930957495e-06,
+      "loss": 0.7314,
+      "step": 1240
+    },
+    {
+      "epoch": 0.48132460531382365,
+      "grad_norm": 0.17140258848667145,
+      "learning_rate": 5.550747882748095e-06,
+      "loss": 0.7233,
+      "step": 1250
+    },
+    {
+      "epoch": 0.48517520215633425,
+      "grad_norm": 0.21700510382652283,
+      "learning_rate": 5.4887281720638195e-06,
+      "loss": 0.7369,
+      "step": 1260
+    },
+    {
+      "epoch": 0.4890257989988448,
+      "grad_norm": 0.19014368951320648,
+      "learning_rate": 5.42663244535466e-06,
+      "loss": 0.713,
+      "step": 1270
+    },
+    {
+      "epoch": 0.4928763958413554,
+      "grad_norm": 0.18108373880386353,
+      "learning_rate": 5.3644703608940205e-06,
+      "loss": 0.7317,
+      "step": 1280
+    },
+    {
+      "epoch": 0.496726992683866,
+      "grad_norm": 0.16324463486671448,
+      "learning_rate": 5.302251587276491e-06,
+      "loss": 0.743,
+      "step": 1290
+    },
+    {
+      "epoch": 0.5005775895263765,
+      "grad_norm": 0.17062409222126007,
+      "learning_rate": 5.2399858019140005e-06,
+      "loss": 0.7331,
+      "step": 1300
+    },
+    {
+      "epoch": 0.5044281863688872,
+      "grad_norm": 0.1837308257818222,
+      "learning_rate": 5.177682689530615e-06,
+      "loss": 0.7532,
+      "step": 1310
+    },
+    {
+      "epoch": 0.5082787832113977,
+      "grad_norm": 0.18153858184814453,
+      "learning_rate": 5.11535194065619e-06,
+      "loss": 0.7228,
+      "step": 1320
+    },
+    {
+      "epoch": 0.5121293800539084,
+      "grad_norm": 0.18230943381786346,
+      "learning_rate": 5.053003250119113e-06,
+      "loss": 0.7311,
+      "step": 1330
+    },
+    {
+      "epoch": 0.515979976896419,
+      "grad_norm": 0.18009211122989655,
+      "learning_rate": 4.9906463155383925e-06,
+      "loss": 0.7372,
+      "step": 1340
+    },
+    {
+      "epoch": 0.5198305737389295,
+      "grad_norm": 0.17651905119419098,
+      "learning_rate": 4.9282908358153046e-06,
+      "loss": 0.7358,
+      "step": 1350
+    },
+    {
+      "epoch": 0.5236811705814401,
+      "grad_norm": 0.18802909553050995,
+      "learning_rate": 4.8659465096248345e-06,
+      "loss": 0.7494,
+      "step": 1360
+    },
+    {
+      "epoch": 0.5275317674239507,
+      "grad_norm": 0.18198397755622864,
+      "learning_rate": 4.803623033907169e-06,
+      "loss": 0.7418,
+      "step": 1370
+    },
+    {
+      "epoch": 0.5313823642664613,
+      "grad_norm": 0.16924996674060822,
+      "learning_rate": 4.741330102359441e-06,
+      "loss": 0.7548,
+      "step": 1380
+    },
+    {
+      "epoch": 0.5352329611089719,
+      "grad_norm": 0.18025757372379303,
+      "learning_rate": 4.679077403927996e-06,
+      "loss": 0.7302,
+      "step": 1390
+    },
+    {
+      "epoch": 0.5390835579514824,
+      "grad_norm": 0.17740656435489655,
+      "learning_rate": 4.616874621301373e-06,
+      "loss": 0.7324,
+      "step": 1400
+    },
+    {
+      "epoch": 0.5429341547939931,
+      "grad_norm": 0.18277397751808167,
+      "learning_rate": 4.554731429404293e-06,
+      "loss": 0.7512,
+      "step": 1410
+    },
+    {
+      "epoch": 0.5467847516365036,
+      "grad_norm": 0.18542250990867615,
+      "learning_rate": 4.492657493892822e-06,
+      "loss": 0.7329,
+      "step": 1420
+    },
+    {
+      "epoch": 0.5506353484790143,
+      "grad_norm": 0.1805475354194641,
+      "learning_rate": 4.430662469650999e-06,
+      "loss": 0.746,
+      "step": 1430
+    },
+    {
+      "epoch": 0.5544859453215248,
+      "grad_norm": 0.1896904557943344,
+      "learning_rate": 4.368755999289118e-06,
+      "loss": 0.7481,
+      "step": 1440
+    },
+    {
+      "epoch": 0.5583365421640354,
+      "grad_norm": 0.18090938031673431,
+      "learning_rate": 4.306947711643949e-06,
+      "loss": 0.7317,
+      "step": 1450
+    },
+    {
+      "epoch": 0.562187139006546,
+      "grad_norm": 0.17698980867862701,
+      "learning_rate": 4.245247220281068e-06,
+      "loss": 0.7367,
+      "step": 1460
+    },
+    {
+      "epoch": 0.5660377358490566,
+      "grad_norm": 0.17120298743247986,
+      "learning_rate": 4.183664121999587e-06,
+      "loss": 0.7318,
+      "step": 1470
+    },
+    {
+      "epoch": 0.5698883326915672,
+      "grad_norm": 0.1741509735584259,
+      "learning_rate": 4.122207995339474e-06,
+      "loss": 0.7266,
+      "step": 1480
+    },
+    {
+      "epoch": 0.5737389295340778,
+      "grad_norm": 0.17797298729419708,
+      "learning_rate": 4.060888399091734e-06,
+      "loss": 0.7345,
+      "step": 1490
+    },
+    {
+      "epoch": 0.5775895263765883,
+      "grad_norm": 0.17725734412670135,
+      "learning_rate": 3.999714870811639e-06,
+      "loss": 0.7467,
+      "step": 1500
+    },
+    {
+      "epoch": 0.581440123219099,
+      "grad_norm": 0.17090828716754913,
+      "learning_rate": 3.938696925335275e-06,
+      "loss": 0.74,
+      "step": 1510
+    },
+    {
+      "epoch": 0.5852907200616095,
+      "grad_norm": 0.1822526454925537,
+      "learning_rate": 3.87784405329962e-06,
+      "loss": 0.7466,
+      "step": 1520
+    },
+    {
+      "epoch": 0.5891413169041202,
+      "grad_norm": 0.16938379406929016,
+      "learning_rate": 3.817165719666382e-06,
+      "loss": 0.7473,
+      "step": 1530
+    },
+    {
+      "epoch": 0.5929919137466307,
+      "grad_norm": 0.17732830345630646,
+      "learning_rate": 3.756671362249841e-06,
+      "loss": 0.7503,
+      "step": 1540
+    },
+    {
+      "epoch": 0.5968425105891413,
+      "grad_norm": 0.16901449859142303,
+      "learning_rate": 3.6963703902488907e-06,
+      "loss": 0.7375,
+      "step": 1550
+    },
+    {
+      "epoch": 0.6006931074316519,
+      "grad_norm": 0.1684798300266266,
+      "learning_rate": 3.6362721827835655e-06,
+      "loss": 0.7477,
+      "step": 1560
+    },
+    {
+      "epoch": 0.6045437042741625,
+      "grad_norm": 0.18177856504917145,
+      "learning_rate": 3.576386087436212e-06,
+      "loss": 0.7544,
+      "step": 1570
+    },
+    {
+      "epoch": 0.608394301116673,
+      "grad_norm": 0.18912944197654724,
+      "learning_rate": 3.5167214187975913e-06,
+      "loss": 0.7458,
+      "step": 1580
+    },
+    {
+      "epoch": 0.6122448979591837,
+      "grad_norm": 0.15865333378314972,
+      "learning_rate": 3.457287457018086e-06,
+      "loss": 0.7416,
+      "step": 1590
+    },
+    {
+      "epoch": 0.6160954948016942,
+      "grad_norm": 0.17447324097156525,
+      "learning_rate": 3.3980934463643003e-06,
+      "loss": 0.737,
+      "step": 1600
+    },
+    {
+      "epoch": 0.6199460916442049,
+      "grad_norm": 0.1629418581724167,
+      "learning_rate": 3.3391485937812053e-06,
+      "loss": 0.7154,
+      "step": 1610
+    },
+    {
+      "epoch": 0.6237966884867154,
+      "grad_norm": 0.18057814240455627,
+      "learning_rate": 3.280462067460115e-06,
+      "loss": 0.716,
+      "step": 1620
+    },
+    {
+      "epoch": 0.627647285329226,
+      "grad_norm": 0.17002978920936584,
+      "learning_rate": 3.222042995412669e-06,
+      "loss": 0.7416,
+      "step": 1630
+    },
+    {
+      "epoch": 0.6314978821717366,
+      "grad_norm": 0.16219274699687958,
+      "learning_rate": 3.163900464051089e-06,
+      "loss": 0.7335,
+      "step": 1640
+    },
+    {
+      "epoch": 0.6353484790142472,
+      "grad_norm": 0.18324173986911774,
+      "learning_rate": 3.1060435167748826e-06,
+      "loss": 0.7489,
+      "step": 1650
+    },
+    {
+      "epoch": 0.6391990758567578,
+      "grad_norm": 0.17670175433158875,
+      "learning_rate": 3.048481152564253e-06,
+      "loss": 0.7239,
+      "step": 1660
+    },
+    {
+      "epoch": 0.6430496726992684,
+      "grad_norm": 0.18382449448108673,
+      "learning_rate": 2.9912223245804017e-06,
+      "loss": 0.743,
+      "step": 1670
+    },
+    {
+      "epoch": 0.6469002695417789,
+      "grad_norm": 0.17219989001750946,
+      "learning_rate": 2.934275938772981e-06,
+      "loss": 0.7346,
+      "step": 1680
+    },
+    {
+      "epoch": 0.6507508663842896,
+      "grad_norm": 0.18355175852775574,
+      "learning_rate": 2.8776508524948676e-06,
+      "loss": 0.73,
+      "step": 1690
+    },
+    {
+      "epoch": 0.6546014632268001,
+      "grad_norm": 0.16541482508182526,
+      "learning_rate": 2.821355873124503e-06,
+      "loss": 0.7316,
+      "step": 1700
+    },
+    {
+      "epoch": 0.6584520600693108,
+      "grad_norm": 0.1847158968448639,
+      "learning_rate": 2.7653997566960144e-06,
+      "loss": 0.7379,
+      "step": 1710
+    },
+    {
+      "epoch": 0.6623026569118213,
+      "grad_norm": 0.18662187457084656,
+      "learning_rate": 2.709791206537309e-06,
+      "loss": 0.7312,
+      "step": 1720
+    },
+    {
+      "epoch": 0.6661532537543319,
+      "grad_norm": 0.19208745658397675,
+      "learning_rate": 2.6545388719163747e-06,
+      "loss": 0.7362,
+      "step": 1730
+    },
+    {
+      "epoch": 0.6700038505968425,
+      "grad_norm": 0.17661386728286743,
+      "learning_rate": 2.599651346695979e-06,
+      "loss": 0.7443,
+      "step": 1740
+    },
+    {
+      "epoch": 0.6738544474393531,
+      "grad_norm": 0.16614247858524323,
+      "learning_rate": 2.545137167997007e-06,
+      "loss": 0.7388,
+      "step": 1750
+    },
+    {
+      "epoch": 0.6777050442818637,
+      "grad_norm": 0.16882900893688202,
+      "learning_rate": 2.4910048148705972e-06,
+      "loss": 0.7303,
+      "step": 1760
+    },
+    {
+      "epoch": 0.6815556411243743,
+      "grad_norm": 0.18148168921470642,
+      "learning_rate": 2.4372627069793327e-06,
+      "loss": 0.7677,
+      "step": 1770
+    },
+    {
+      "epoch": 0.6854062379668848,
+      "grad_norm": 0.17098696529865265,
+      "learning_rate": 2.3839192032876486e-06,
+      "loss": 0.7432,
+      "step": 1780
+    },
+    {
+      "epoch": 0.6892568348093955,
+      "grad_norm": 0.17942175269126892,
+      "learning_rate": 2.3309826007617047e-06,
+      "loss": 0.7587,
+      "step": 1790
+    },
+    {
+      "epoch": 0.693107431651906,
+      "grad_norm": 0.17586492002010345,
+      "learning_rate": 2.278461133078881e-06,
+      "loss": 0.728,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6969580284944167,
+      "grad_norm": 0.18195484578609467,
+      "learning_rate": 2.2263629693471258e-06,
+      "loss": 0.7211,
+      "step": 1810
+    },
+    {
+      "epoch": 0.7008086253369272,
+      "grad_norm": 0.16105888783931732,
+      "learning_rate": 2.1746962128343414e-06,
+      "loss": 0.7434,
+      "step": 1820
+    },
+    {
+      "epoch": 0.7046592221794378,
+      "grad_norm": 0.1759762018918991,
+      "learning_rate": 2.1234688997080183e-06,
+      "loss": 0.7274,
+      "step": 1830
+    },
+    {
+      "epoch": 0.7085098190219484,
+      "grad_norm": 0.17354726791381836,
+      "learning_rate": 2.0726889977853053e-06,
+      "loss": 0.7331,
+      "step": 1840
+    },
+    {
+      "epoch": 0.712360415864459,
+      "grad_norm": 0.18378381431102753,
+      "learning_rate": 2.022364405293703e-06,
+      "loss": 0.7473,
+      "step": 1850
+    },
+    {
+      "epoch": 0.7162110127069696,
+      "grad_norm": 0.17597711086273193,
+      "learning_rate": 1.9725029496425814e-06,
+      "loss": 0.7456,
+      "step": 1860
+    },
+    {
+      "epoch": 0.7200616095494802,
+      "grad_norm": 0.17941486835479736,
+      "learning_rate": 1.923112386205727e-06,
+      "loss": 0.7357,
+      "step": 1870
+    },
+    {
+      "epoch": 0.7239122063919907,
+      "grad_norm": 0.17042817175388336,
+      "learning_rate": 1.874200397115079e-06,
+      "loss": 0.7469,
+      "step": 1880
+    },
+    {
+      "epoch": 0.7277628032345014,
+      "grad_norm": 0.17936168611049652,
+      "learning_rate": 1.825774590065857e-06,
+      "loss": 0.7319,
+      "step": 1890
+    },
+    {
+      "epoch": 0.7316134000770119,
+      "grad_norm": 0.18507741391658783,
+      "learning_rate": 1.7778424971332813e-06,
+      "loss": 0.7437,
+      "step": 1900
+    },
+    {
+      "epoch": 0.7354639969195226,
+      "grad_norm": 0.16395284235477448,
+      "learning_rate": 1.7304115736010407e-06,
+      "loss": 0.739,
+      "step": 1910
+    },
+    {
+      "epoch": 0.7393145937620331,
+      "grad_norm": 0.17336514592170715,
+      "learning_rate": 1.6834891968017108e-06,
+      "loss": 0.7303,
+      "step": 1920
+    },
+    {
+      "epoch": 0.7431651906045437,
+      "grad_norm": 0.19245952367782593,
+      "learning_rate": 1.6370826649692884e-06,
+      "loss": 0.7482,
+      "step": 1930
+    },
+    {
+      "epoch": 0.7470157874470543,
+      "grad_norm": 0.169088676571846,
+      "learning_rate": 1.5911991961040463e-06,
+      "loss": 0.7472,
+      "step": 1940
+    },
+    {
+      "epoch": 0.7508663842895649,
+      "grad_norm": 0.17976053059101105,
+      "learning_rate": 1.545845926849851e-06,
+      "loss": 0.7299,
+      "step": 1950
+    },
+    {
+      "epoch": 0.7547169811320755,
+      "grad_norm": 0.18894024193286896,
+      "learning_rate": 1.5010299113841397e-06,
+      "loss": 0.7547,
+      "step": 1960
+    },
+    {
+      "epoch": 0.7585675779745861,
+      "grad_norm": 0.18768291175365448,
+      "learning_rate": 1.4567581203207238e-06,
+      "loss": 0.7453,
+      "step": 1970
+    },
+    {
+      "epoch": 0.7624181748170966,
+      "grad_norm": 0.19523227214813232,
+      "learning_rate": 1.413037439625593e-06,
+      "loss": 0.7444,
+      "step": 1980
+    },
+    {
+      "epoch": 0.7662687716596073,
+      "grad_norm": 0.1881563514471054,
+      "learning_rate": 1.3698746695458847e-06,
+      "loss": 0.7337,
+      "step": 1990
+    },
+    {
+      "epoch": 0.7701193685021178,
+      "grad_norm": 0.1758456975221634,
+      "learning_rate": 1.3272765235521745e-06,
+      "loss": 0.7348,
+      "step": 2000
+    },
+    {
+      "epoch": 0.7739699653446284,
+      "grad_norm": 0.1732279360294342,
+      "learning_rate": 1.2852496272942877e-06,
+      "loss": 0.7424,
+      "step": 2010
+    },
+    {
+      "epoch": 0.777820562187139,
+      "grad_norm": 0.16616225242614746,
+      "learning_rate": 1.2438005175707396e-06,
+      "loss": 0.7239,
+      "step": 2020
+    },
+    {
+      "epoch": 0.7816711590296496,
+      "grad_norm": 0.17293043434619904,
+      "learning_rate": 1.2029356413120297e-06,
+      "loss": 0.7175,
+      "step": 2030
+    },
+    {
+      "epoch": 0.7855217558721602,
+      "grad_norm": 0.1762024462223053,
+      "learning_rate": 1.1626613545778786e-06,
+      "loss": 0.7322,
+      "step": 2040
+    },
+    {
+      "epoch": 0.7893723527146708,
+      "grad_norm": 0.16409577429294586,
+      "learning_rate": 1.1229839215686294e-06,
+      "loss": 0.7355,
+      "step": 2050
+    },
+    {
+      "epoch": 0.7932229495571813,
+      "grad_norm": 0.15731360018253326,
+      "learning_rate": 1.0839095136509188e-06,
+      "loss": 0.7198,
+      "step": 2060
+    },
+    {
+      "epoch": 0.797073546399692,
+      "grad_norm": 0.17766538262367249,
+      "learning_rate": 1.045444208397791e-06,
+      "loss": 0.7365,
+      "step": 2070
+    },
+    {
+      "epoch": 0.8009241432422025,
+      "grad_norm": 0.16605044901371002,
+      "learning_rate": 1.007593988643402e-06,
+      "loss": 0.7132,
+      "step": 2080
+    },
+    {
+      "epoch": 0.8047747400847132,
+      "grad_norm": 0.16463790833950043,
+      "learning_rate": 9.703647415524613e-07,
+      "loss": 0.7414,
+      "step": 2090
+    },
+    {
+      "epoch": 0.8086253369272237,
+      "grad_norm": 0.1649317741394043,
+      "learning_rate": 9.337622577045485e-07,
+      "loss": 0.7323,
+      "step": 2100
+    },
+    {
+      "epoch": 0.8124759337697343,
+      "grad_norm": 0.17177657783031464,
+      "learning_rate": 8.977922301934588e-07,
+      "loss": 0.7361,
+      "step": 2110
+    },
+    {
+      "epoch": 0.8163265306122449,
+      "grad_norm": 0.18438002467155457,
+      "learning_rate": 8.624602537417009e-07,
+      "loss": 0.7503,
+      "step": 2120
+    },
+    {
+      "epoch": 0.8201771274547555,
+      "grad_norm": 0.1705552339553833,
+      "learning_rate": 8.277718238303123e-07,
+      "loss": 0.7321,
+      "step": 2130
+    },
+    {
+      "epoch": 0.8240277242972661,
+      "grad_norm": 0.18389014899730682,
+      "learning_rate": 7.937323358440935e-07,
+      "loss": 0.7343,
+      "step": 2140
+    },
+    {
+      "epoch": 0.8278783211397767,
+      "grad_norm": 0.17136543989181519,
+      "learning_rate": 7.603470842324223e-07,
+      "loss": 0.7226,
+      "step": 2150
+    },
+    {
+      "epoch": 0.8317289179822872,
+      "grad_norm": 0.16815848648548126,
+      "learning_rate": 7.276212616857575e-07,
+      "loss": 0.7446,
+      "step": 2160
+    },
+    {
+      "epoch": 0.8355795148247979,
+      "grad_norm": 0.17567980289459229,
+      "learning_rate": 6.955599583279837e-07,
+      "loss": 0.7372,
+      "step": 2170
+    },
+    {
+      "epoch": 0.8394301116673084,
+      "grad_norm": 0.166676327586174,
+      "learning_rate": 6.641681609246981e-07,
+      "loss": 0.734,
+      "step": 2180
+    },
+    {
+      "epoch": 0.8432807085098191,
+      "grad_norm": 0.1773717701435089,
+      "learning_rate": 6.334507521075711e-07,
+      "loss": 0.733,
+      "step": 2190
+    },
+    {
+      "epoch": 0.8471313053523296,
+      "grad_norm": 0.18327943980693817,
+      "learning_rate": 6.034125096149179e-07,
+      "loss": 0.7361,
+      "step": 2200
+    },
+    {
+      "epoch": 0.8509819021948402,
+      "grad_norm": 0.17350366711616516,
+      "learning_rate": 5.740581055485722e-07,
+      "loss": 0.7079,
+      "step": 2210
+    },
+    {
+      "epoch": 0.8548324990373508,
+      "grad_norm": 0.16288484632968903,
+      "learning_rate": 5.453921056471961e-07,
+      "loss": 0.7398,
+      "step": 2220
+    },
+    {
+      "epoch": 0.8586830958798614,
+      "grad_norm": 0.185551255941391,
+      "learning_rate": 5.174189685761288e-07,
+      "loss": 0.7345,
+      "step": 2230
+    },
+    {
+      "epoch": 0.862533692722372,
+      "grad_norm": 0.1667529046535492,
+      "learning_rate": 4.901430452338979e-07,
+      "loss": 0.7455,
+      "step": 2240
+    },
+    {
+      "epoch": 0.8663842895648826,
+      "grad_norm": 0.1693091094493866,
+      "learning_rate": 4.6356857807548286e-07,
+      "loss": 0.732,
+      "step": 2250
+    },
+    {
+      "epoch": 0.8702348864073931,
+      "grad_norm": 0.19884486496448517,
+      "learning_rate": 4.376997004524541e-07,
+      "loss": 0.7161,
+      "step": 2260
+    },
+    {
+      "epoch": 0.8740854832499038,
+      "grad_norm": 0.16969723999500275,
+      "learning_rate": 4.12540435970073e-07,
+      "loss": 0.733,
+      "step": 2270
+    },
+    {
+      "epoch": 0.8779360800924143,
+      "grad_norm": 0.16635586321353912,
+      "learning_rate": 3.880946978614703e-07,
+      "loss": 0.7408,
+      "step": 2280
+    },
+    {
+      "epoch": 0.881786676934925,
+      "grad_norm": 0.16847485303878784,
+      "learning_rate": 3.643662883789878e-07,
+      "loss": 0.7419,
+      "step": 2290
+    },
+    {
+      "epoch": 0.8856372737774355,
+      "grad_norm": 0.16418440639972687,
+      "learning_rate": 3.41358898202781e-07,
+      "loss": 0.7246,
+      "step": 2300
+    },
+    {
+      "epoch": 0.889487870619946,
+      "grad_norm": 0.16536955535411835,
+      "learning_rate": 3.190761058667741e-07,
+      "loss": 0.7358,
+      "step": 2310
+    },
+    {
+      "epoch": 0.8933384674624567,
+      "grad_norm": 0.16429728269577026,
+      "learning_rate": 2.9752137720206463e-07,
+      "loss": 0.7369,
+      "step": 2320
+    },
+    {
+      "epoch": 0.8971890643049673,
+      "grad_norm": 0.1767251044511795,
+      "learning_rate": 2.7669806479785067e-07,
+      "loss": 0.7477,
+      "step": 2330
+    },
+    {
+      "epoch": 0.9010396611474779,
+      "grad_norm": 0.1645691841840744,
+      "learning_rate": 2.566094074799763e-07,
+      "loss": 0.7387,
+      "step": 2340
+    },
+    {
+      "epoch": 0.9048902579899885,
+      "grad_norm": 0.17867755889892578,
+      "learning_rate": 2.3725852980716557e-07,
+      "loss": 0.72,
+      "step": 2350
+    },
+    {
+      "epoch": 0.908740854832499,
+      "grad_norm": 0.16647519171237946,
+      "learning_rate": 2.1864844158503983e-07,
+      "loss": 0.7349,
+      "step": 2360
+    },
+    {
+      "epoch": 0.9125914516750097,
+      "grad_norm": 0.1867886632680893,
+      "learning_rate": 2.007820373979702e-07,
+      "loss": 0.7278,
+      "step": 2370
+    },
+    {
+      "epoch": 0.9164420485175202,
+      "grad_norm": 0.16669507324695587,
+      "learning_rate": 1.8366209615886532e-07,
+      "loss": 0.7384,
+      "step": 2380
+    },
+    {
+      "epoch": 0.9202926453600309,
+      "grad_norm": 0.17427407205104828,
+      "learning_rate": 1.6729128067693613e-07,
+      "loss": 0.7513,
+      "step": 2390
+    },
+    {
+      "epoch": 0.9241432422025414,
+      "grad_norm": 0.18462347984313965,
+      "learning_rate": 1.5167213724353426e-07,
+      "loss": 0.7448,
+      "step": 2400
+    },
+    {
+      "epoch": 0.927993839045052,
+      "grad_norm": 0.1734476387500763,
+      "learning_rate": 1.3680709523610058e-07,
+      "loss": 0.7234,
+      "step": 2410
+    },
+    {
+      "epoch": 0.9318444358875626,
+      "grad_norm": 0.16484853625297546,
+      "learning_rate": 1.2269846674030418e-07,
+      "loss": 0.7522,
+      "step": 2420
+    },
+    {
+      "epoch": 0.9356950327300732,
+      "grad_norm": 0.17095895111560822,
+      "learning_rate": 1.0934844619042595e-07,
+      "loss": 0.7456,
+      "step": 2430
+    },
+    {
+      "epoch": 0.9395456295725837,
+      "grad_norm": 0.17096492648124695,
+      "learning_rate": 9.675911002803961e-08,
+      "loss": 0.7344,
+      "step": 2440
+    },
+    {
+      "epoch": 0.9433962264150944,
+      "grad_norm": 0.17844420671463013,
+      "learning_rate": 8.493241637904381e-08,
+      "loss": 0.7227,
+      "step": 2450
+    },
+    {
+      "epoch": 0.9472468232576049,
+      "grad_norm": 0.18762044608592987,
+      "learning_rate": 7.387020474909689e-08,
+      "loss": 0.7489,
+      "step": 2460
+    },
+    {
+      "epoch": 0.9510974201001156,
+      "grad_norm": 0.18394458293914795,
+      "learning_rate": 6.357419573750856e-08,
+      "loss": 0.7449,
+      "step": 2470
+    },
+    {
+      "epoch": 0.9549480169426261,
+      "grad_norm": 0.17509901523590088,
+      "learning_rate": 5.4045990769613365e-08,
+      "loss": 0.7312,
+      "step": 2480
+    },
+    {
+      "epoch": 0.9587986137851366,
+      "grad_norm": 0.17106319963932037,
+      "learning_rate": 4.5287071847694434e-08,
+      "loss": 0.733,
+      "step": 2490
+    },
+    {
+      "epoch": 0.9626492106276473,
+      "grad_norm": 0.1677611619234085,
+      "learning_rate": 3.7298801320467817e-08,
+      "loss": 0.7356,
+      "step": 2500
+    },
+    {
+      "epoch": 0.9664998074701578,
+      "grad_norm": 0.1675986796617508,
+      "learning_rate": 3.0082421671192576e-08,
+      "loss": 0.7347,
+      "step": 2510
+    },
+    {
+      "epoch": 0.9703504043126685,
+      "grad_norm": 0.17156146466732025,
+      "learning_rate": 2.3639055324410355e-08,
+      "loss": 0.7488,
+      "step": 2520
+    },
+    {
+      "epoch": 0.974201001155179,
+      "grad_norm": 0.17610937356948853,
+      "learning_rate": 1.7969704471369477e-08,
+      "loss": 0.7286,
+      "step": 2530
+    },
+    {
+      "epoch": 0.9780515979976896,
+      "grad_norm": 0.16561666131019592,
+      "learning_rate": 1.3075250914142413e-08,
+      "loss": 0.729,
+      "step": 2540
+    },
+    {
+      "epoch": 0.9819021948402002,
+      "grad_norm": 0.16974012553691864,
+      "learning_rate": 8.956455928476049e-09,
+      "loss": 0.7446,
+      "step": 2550
+    },
+    {
+      "epoch": 0.9857527916827108,
+      "grad_norm": 0.16223949193954468,
+      "learning_rate": 5.6139601453786275e-09,
+      "loss": 0.7295,
+      "step": 2560
+    },
+    {
+      "epoch": 0.9896033885252215,
+      "grad_norm": 0.16692401468753815,
+      "learning_rate": 3.0482834514805602e-09,
+      "loss": 0.7253,
+      "step": 2570
+    },
+    {
+      "epoch": 0.993453985367732,
+      "grad_norm": 0.1676122099161148,
+      "learning_rate": 1.2598249081713366e-09,
+      "loss": 0.7157,
+      "step": 2580
+    },
+    {
+      "epoch": 0.9973045822102425,
+      "grad_norm": 0.1831001192331314,
+      "learning_rate": 2.488626895302826e-10,
+      "loss": 0.7496,
+      "step": 2590
+    },
+    {
+      "epoch": 1.0,
+      "step": 2597,
+      "total_flos": 6.731071331099476e+18,
+      "train_loss": 0.7505866338254674,
+      "train_runtime": 14861.4451,
+      "train_samples_per_second": 44.74,
+      "train_steps_per_second": 0.175
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2597,
+  "num_input_tokens_seen": 584317136,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.731071331099476e+18,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

continuous/smollm2/clip_openai__l14/finetune/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:464b67cfc7e14419d0646e36d3a9fb7a3c7d0c8fffdb23ad795314de3165fa4a
+size 6264

continuous/smollm2/clip_openai__l14/pretrain/config.json

@@ -0,0 +1,70 @@
+{
+  "add_faster_video": false,
+  "add_time_instruction": false,
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "faster_token_stride": 10,
+  "force_sample": false,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_aspect_ratio": "square",
+  "image_crop_resolution": null,
+  "image_grid_pinpoints": null,
+  "image_split_resolution": null,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mm_hidden_size": 1024,
+  "mm_newline_position": "grid",
+  "mm_patch_merge_type": "flat",
+  "mm_projector_lr": null,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_resampler_type": null,
+  "mm_spatial_pool_mode": "bilinear",
+  "mm_spatial_pool_stride": null,
+  "mm_tunable_parts": "mm_mlp_adapter",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -2,
+  "mm_vision_tower": "/cache/ckpt/download/tokenizer/continuous/clip-vit-large-patch14",
+  "mm_vision_tower_lr": null,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "pos_skipping_range": 4096,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 130000,
+  "tie_word_embeddings": true,
+  "tokenizer_model_max_length": 8192,
+  "tokenizer_padding_side": "right",
+  "transformers.js_config": {
+    "dtype": "q4",
+    "kv_cache_dtype": {
+      "fp16": "float16",
+      "q4f16": "float16"
+    },
+    "use_external_data_format": {
+      "model.onnx": true,
+      "model_fp16.onnx": true
+    }
+  },
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "use_mm_proj": true,
+  "use_pos_skipping": false,
+  "vision_tower_pretrained": null,
+  "vocab_size": 49152
+}

continuous/smollm2/clip_openai__l14/pretrain/mm_projector.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d5b63b8d9a61cd2cd05e29d11f0a7d923cdab0ed67cee58903ef1a56a43ef05
+size 25184312

continuous/smollm2/clip_openai__l14/pretrain/trainer_state.json

@@ -0,0 +1,421 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 545,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01834862385321101,
+      "grad_norm": 2.3370816707611084,
+      "learning_rate": 0.00032142857142857147,
+      "loss": 3.6863,
+      "step": 10
+    },
+    {
+      "epoch": 0.03669724770642202,
+      "grad_norm": 0.28958168625831604,
+      "learning_rate": 0.0006785714285714287,
+      "loss": 2.9871,
+      "step": 20
+    },
+    {
+      "epoch": 0.05504587155963303,
+      "grad_norm": 0.15311428904533386,
+      "learning_rate": 0.0009999907688176173,
+      "loss": 2.7983,
+      "step": 30
+    },
+    {
+      "epoch": 0.07339449541284404,
+      "grad_norm": 0.1466771513223648,
+      "learning_rate": 0.0009988834393115766,
+      "loss": 2.7009,
+      "step": 40
+    },
+    {
+      "epoch": 0.09174311926605505,
+      "grad_norm": 0.10041819512844086,
+      "learning_rate": 0.0009959345572940772,
+      "loss": 2.6173,
+      "step": 50
+    },
+    {
+      "epoch": 0.11009174311926606,
+      "grad_norm": 0.13481402397155762,
+      "learning_rate": 0.0009911550081155983,
+      "loss": 2.5393,
+      "step": 60
+    },
+    {
+      "epoch": 0.12844036697247707,
+      "grad_norm": 0.1153503954410553,
+      "learning_rate": 0.0009845624347566433,
+      "loss": 2.4864,
+      "step": 70
+    },
+    {
+      "epoch": 0.14678899082568808,
+      "grad_norm": 0.11440035700798035,
+      "learning_rate": 0.0009761811727013547,
+      "loss": 2.44,
+      "step": 80
+    },
+    {
+      "epoch": 0.1651376146788991,
+      "grad_norm": 0.09977620095014572,
+      "learning_rate": 0.0009660421601067666,
+      "loss": 2.4039,
+      "step": 90
+    },
+    {
+      "epoch": 0.1834862385321101,
+      "grad_norm": 0.1337294578552246,
+      "learning_rate": 0.0009541828235992926,
+      "loss": 2.3826,
+      "step": 100
+    },
+    {
+      "epoch": 0.2018348623853211,
+      "grad_norm": 0.11631788313388824,
+      "learning_rate": 0.0009406469401200151,
+      "loss": 2.3455,
+      "step": 110
+    },
+    {
+      "epoch": 0.22018348623853212,
+      "grad_norm": 0.10597465187311172,
+      "learning_rate": 0.0009254844753287492,
+      "loss": 2.3194,
+      "step": 120
+    },
+    {
+      "epoch": 0.23853211009174313,
+      "grad_norm": 0.10810327529907227,
+      "learning_rate": 0.0009087513991633924,
+      "loss": 2.2814,
+      "step": 130
+    },
+    {
+      "epoch": 0.25688073394495414,
+      "grad_norm": 0.14619243144989014,
+      "learning_rate": 0.0008905094792353917,
+      "loss": 2.2631,
+      "step": 140
+    },
+    {
+      "epoch": 0.27522935779816515,
+      "grad_norm": 0.13312995433807373,
+      "learning_rate": 0.0008708260528239789,
+      "loss": 2.2578,
+      "step": 150
+    },
+    {
+      "epoch": 0.29357798165137616,
+      "grad_norm": 0.1467946469783783,
+      "learning_rate": 0.0008497737783108237,
+      "loss": 2.2373,
+      "step": 160
+    },
+    {
+      "epoch": 0.3119266055045872,
+      "grad_norm": 0.10098150372505188,
+      "learning_rate": 0.0008274303669726426,
+      "loss": 2.2331,
+      "step": 170
+    },
+    {
+      "epoch": 0.3302752293577982,
+      "grad_norm": 0.10856643319129944,
+      "learning_rate": 0.0008038782961218137,
+      "loss": 2.2215,
+      "step": 180
+    },
+    {
+      "epoch": 0.3486238532110092,
+      "grad_norm": 0.10588429123163223,
+      "learning_rate": 0.000779204504653894,
+      "loss": 2.2142,
+      "step": 190
+    },
+    {
+      "epoch": 0.3669724770642202,
+      "grad_norm": 0.10607463121414185,
+      "learning_rate": 0.0007535000721258791,
+      "loss": 2.1896,
+      "step": 200
+    },
+    {
+      "epoch": 0.3853211009174312,
+      "grad_norm": 0.11839769035577774,
+      "learning_rate": 0.00072685988254984,
+      "loss": 2.1856,
+      "step": 210
+    },
+    {
+      "epoch": 0.4036697247706422,
+      "grad_norm": 0.11571677774190903,
+      "learning_rate": 0.0006993822741429906,
+      "loss": 2.1929,
+      "step": 220
+    },
+    {
+      "epoch": 0.42201834862385323,
+      "grad_norm": 0.12334726750850677,
+      "learning_rate": 0.0006711686763270818,
+      "loss": 2.1816,
+      "step": 230
+    },
+    {
+      "epoch": 0.44036697247706424,
+      "grad_norm": 0.09982427954673767,
+      "learning_rate": 0.0006423232353170831,
+      "loss": 2.1755,
+      "step": 240
+    },
+    {
+      "epoch": 0.45871559633027525,
+      "grad_norm": 0.08938395977020264,
+      "learning_rate": 0.0006129524296812335,
+      "loss": 2.1778,
+      "step": 250
+    },
+    {
+      "epoch": 0.47706422018348627,
+      "grad_norm": 0.09739397466182709,
+      "learning_rate": 0.0005831646772915651,
+      "loss": 2.18,
+      "step": 260
+    },
+    {
+      "epoch": 0.4954128440366973,
+      "grad_norm": 0.09275171905755997,
+      "learning_rate": 0.0005530699351157809,
+      "loss": 2.1593,
+      "step": 270
+    },
+    {
+      "epoch": 0.5137614678899083,
+      "grad_norm": 0.10106043517589569,
+      "learning_rate": 0.0005227792933277943,
+      "loss": 2.1632,
+      "step": 280
+    },
+    {
+      "epoch": 0.5321100917431193,
+      "grad_norm": 0.09586358815431595,
+      "learning_rate": 0.0004924045652352092,
+      "loss": 2.1508,
+      "step": 290
+    },
+    {
+      "epoch": 0.5504587155963303,
+      "grad_norm": 0.09010323137044907,
+      "learning_rate": 0.0004620578745374633,
+      "loss": 2.1509,
+      "step": 300
+    },
+    {
+      "epoch": 0.5688073394495413,
+      "grad_norm": 0.12582355737686157,
+      "learning_rate": 0.0004318512414382059,
+      "loss": 2.158,
+      "step": 310
+    },
+    {
+      "epoch": 0.5871559633027523,
+      "grad_norm": 0.10059405118227005,
+      "learning_rate": 0.0004018961691397148,
+      "loss": 2.1576,
+      "step": 320
+    },
+    {
+      "epoch": 0.6055045871559633,
+      "grad_norm": 0.10585736483335495,
+      "learning_rate": 0.0003723032322457458,
+      "loss": 2.1423,
+      "step": 330
+    },
+    {
+      "epoch": 0.6238532110091743,
+      "grad_norm": 0.11908891797065735,
+      "learning_rate": 0.0003431816685921625,
+      "loss": 2.1461,
+      "step": 340
+    },
+    {
+      "epoch": 0.6422018348623854,
+      "grad_norm": 0.10853578895330429,
+      "learning_rate": 0.0003146389760120416,
+      "loss": 2.1461,
+      "step": 350
+    },
+    {
+      "epoch": 0.6605504587155964,
+      "grad_norm": 0.09780111163854599,
+      "learning_rate": 0.00028678051552373485,
+      "loss": 2.1468,
+      "step": 360
+    },
+    {
+      "epoch": 0.6788990825688074,
+      "grad_norm": 0.0898723229765892,
+      "learning_rate": 0.0002597091224066581,
+      "loss": 2.1405,
+      "step": 370
+    },
+    {
+      "epoch": 0.6972477064220184,
+      "grad_norm": 0.09129544347524643,
+      "learning_rate": 0.0002335247266004629,
+      "loss": 2.1472,
+      "step": 380
+    },
+    {
+      "epoch": 0.7155963302752294,
+      "grad_norm": 0.08751478046178818,
+      "learning_rate": 0.000208323983828828,
+      "loss": 2.1368,
+      "step": 390
+    },
+    {
+      "epoch": 0.7339449541284404,
+      "grad_norm": 0.08486471325159073,
+      "learning_rate": 0.0001841999188095224,
+      "loss": 2.1292,
+      "step": 400
+    },
+    {
+      "epoch": 0.7522935779816514,
+      "grad_norm": 0.08976531028747559,
+      "learning_rate": 0.00016124158186777676,
+      "loss": 2.1357,
+      "step": 410
+    },
+    {
+      "epoch": 0.7706422018348624,
+      "grad_norm": 0.09084436297416687,
+      "learning_rate": 0.0001395337202205211,
+      "loss": 2.1312,
+      "step": 420
+    },
+    {
+      "epoch": 0.7889908256880734,
+      "grad_norm": 0.0882534608244896,
+      "learning_rate": 0.00011915646514489292,
+      "loss": 2.1314,
+      "step": 430
+    },
+    {
+      "epoch": 0.8073394495412844,
+      "grad_norm": 0.09434192627668381,
+      "learning_rate": 0.00010018503618578818,
+      "loss": 2.1379,
+      "step": 440
+    },
+    {
+      "epoch": 0.8256880733944955,
+      "grad_norm": 0.08872148394584656,
+      "learning_rate": 8.268946349432583e-05,
+      "loss": 2.1325,
+      "step": 450
+    },
+    {
+      "epoch": 0.8440366972477065,
+      "grad_norm": 0.08502736687660217,
+      "learning_rate": 6.673432932217117e-05,
+      "loss": 2.124,
+      "step": 460
+    },
+    {
+      "epoch": 0.8623853211009175,
+      "grad_norm": 0.0908086821436882,
+      "learning_rate": 5.2378529625954684e-05,
+      "loss": 2.1185,
+      "step": 470
+    },
+    {
+      "epoch": 0.8807339449541285,
+      "grad_norm": 0.08546927571296692,
+      "learning_rate": 3.9675056661785556e-05,
+      "loss": 2.1205,
+      "step": 480
+    },
+    {
+      "epoch": 0.8990825688073395,
+      "grad_norm": 0.08559462428092957,
+      "learning_rate": 2.8670803372379538e-05,
+      "loss": 2.1276,
+      "step": 490
+    },
+    {
+      "epoch": 0.9174311926605505,
+      "grad_norm": 0.0865757092833519,
+      "learning_rate": 1.9406390288876586e-05,
+      "loss": 2.1312,
+      "step": 500
+    },
+    {
+      "epoch": 0.9357798165137615,
+      "grad_norm": 0.08536867797374725,
+      "learning_rate": 1.191601558631461e-05,
+      "loss": 2.1228,
+      "step": 510
+    },
+    {
+      "epoch": 0.9541284403669725,
+      "grad_norm": 0.0846175104379654,
+      "learning_rate": 6.2273288462559305e-06,
+      "loss": 2.1198,
+      "step": 520
+    },
+    {
+      "epoch": 0.9724770642201835,
+      "grad_norm": 0.0863744392991066,
+      "learning_rate": 2.361328992552314e-06,
+      "loss": 2.1235,
+      "step": 530
+    },
+    {
+      "epoch": 0.9908256880733946,
+      "grad_norm": 0.0848388746380806,
+      "learning_rate": 3.322867770044202e-07,
+      "loss": 2.1206,
+      "step": 540
+    },
+    {
+      "epoch": 1.0,
+      "step": 545,
+      "total_flos": 3.99246640352854e+17,
+      "train_loss": 2.2683874777697643,
+      "train_runtime": 1625.8687,
+      "train_samples_per_second": 343.28,
+      "train_steps_per_second": 0.335
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 545,
+  "num_input_tokens_seen": 34658176,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.99246640352854e+17,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

continuous/smollm2/dino_vitb16/finetune/chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

continuous/smollm2/dino_vitb16/finetune/config.json

@@ -0,0 +1,70 @@
+{
+  "add_faster_video": false,
+  "add_time_instruction": false,
+  "architectures": [
+    "LlavaLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "faster_token_stride": 10,
+  "force_sample": false,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_aspect_ratio": "square",
+  "image_crop_resolution": null,
+  "image_grid_pinpoints": null,
+  "image_split_resolution": null,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mm_hidden_size": 768,
+  "mm_newline_position": "grid",
+  "mm_patch_merge_type": "flat",
+  "mm_projector_lr": 0.0001,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_resampler_type": null,
+  "mm_spatial_pool_mode": "bilinear",
+  "mm_spatial_pool_stride": null,
+  "mm_tunable_parts": "mm_mlp_adapter,mm_language_model",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -1,
+  "mm_vision_tower": "hf:/cache/ckpt/download/tokenizer/continuous/dino/dino-vitb16",
+  "mm_vision_tower_lr": null,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "pos_skipping_range": 4096,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 130000,
+  "tie_word_embeddings": true,
+  "tokenizer_model_max_length": 8192,
+  "tokenizer_padding_side": "right",
+  "transformers.js_config": {
+    "dtype": "q4",
+    "kv_cache_dtype": {
+      "fp16": "float16",
+      "q4f16": "float16"
+    },
+    "use_external_data_format": {
+      "model.onnx": true,
+      "model_fp16.onnx": true
+    }
+  },
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "use_mm_proj": true,
+  "use_pos_skipping": false,
+  "vision_tower_pretrained": null,
+  "vocab_size": 49152
+}

continuous/smollm2/dino_vitb16/finetune/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    2
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.57.6"
+}

continuous/smollm2/dino_vitb16/finetune/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9afff8c8af97cdb60fdac25d2157c3495f8a2b1b58aced36a2f4f9bd4b2c69c2
+size 3819998144

continuous/smollm2/dino_vitb16/finetune/special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

continuous/smollm2/dino_vitb16/finetune/tokenizer_config.json

@@ -0,0 +1,155 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

continuous/smollm2/dino_vitb16/finetune/trainer_state.json

@@ -0,0 +1,1856 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 2597,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.003850596842510589,
+      "grad_norm": 3.1716811656951904,
+      "learning_rate": 1.153846153846154e-06,
+      "loss": 1.261,
+      "step": 10
+    },
+    {
+      "epoch": 0.007701193685021178,
+      "grad_norm": 2.174875259399414,
+      "learning_rate": 2.435897435897436e-06,
+      "loss": 1.1549,
+      "step": 20
+    },
+    {
+      "epoch": 0.011551790527531768,
+      "grad_norm": 1.9647698402404785,
+      "learning_rate": 3.7179487179487184e-06,
+      "loss": 1.0255,
+      "step": 30
+    },
+    {
+      "epoch": 0.015402387370042356,
+      "grad_norm": 1.3809078931808472,
+      "learning_rate": 5e-06,
+      "loss": 0.9815,
+      "step": 40
+    },
+    {
+      "epoch": 0.019252984212552945,
+      "grad_norm": 1.2882580757141113,
+      "learning_rate": 6.282051282051282e-06,
+      "loss": 0.9698,
+      "step": 50
+    },
+    {
+      "epoch": 0.023103581055063535,
+      "grad_norm": 0.3128419518470764,
+      "learning_rate": 7.564102564102564e-06,
+      "loss": 0.9025,
+      "step": 60
+    },
+    {
+      "epoch": 0.026954177897574125,
+      "grad_norm": 0.2682138681411743,
+      "learning_rate": 8.846153846153847e-06,
+      "loss": 0.8567,
+      "step": 70
+    },
+    {
+      "epoch": 0.030804774740084712,
+      "grad_norm": 0.2625398635864258,
+      "learning_rate": 9.999996111488723e-06,
+      "loss": 0.8719,
+      "step": 80
+    },
+    {
+      "epoch": 0.0346553715825953,
+      "grad_norm": 0.24373804032802582,
+      "learning_rate": 9.999529497453782e-06,
+      "loss": 0.8598,
+      "step": 90
+    },
+    {
+      "epoch": 0.03850596842510589,
+      "grad_norm": 0.22524020075798035,
+      "learning_rate": 9.998285264324193e-06,
+      "loss": 0.8297,
+      "step": 100
+    },
+    {
+      "epoch": 0.04235656526761648,
+      "grad_norm": 0.227365180850029,
+      "learning_rate": 9.996263605626052e-06,
+      "loss": 0.821,
+      "step": 110
+    },
+    {
+      "epoch": 0.04620716211012707,
+      "grad_norm": 0.23110628128051758,
+      "learning_rate": 9.99346483580503e-06,
+      "loss": 0.8418,
+      "step": 120
+    },
+    {
+      "epoch": 0.05005775895263766,
+      "grad_norm": 0.22075729072093964,
+      "learning_rate": 9.989889390177467e-06,
+      "loss": 0.8188,
+      "step": 130
+    },
+    {
+      "epoch": 0.05390835579514825,
+      "grad_norm": 0.22904381155967712,
+      "learning_rate": 9.985537824862647e-06,
+      "loss": 0.8323,
+      "step": 140
+    },
+    {
+      "epoch": 0.05775895263765884,
+      "grad_norm": 0.24085356295108795,
+      "learning_rate": 9.980410816696317e-06,
+      "loss": 0.8098,
+      "step": 150
+    },
+    {
+      "epoch": 0.061609549480169425,
+      "grad_norm": 0.24090735614299774,
+      "learning_rate": 9.97450916312541e-06,
+      "loss": 0.8149,
+      "step": 160
+    },
+    {
+      "epoch": 0.06546014632268002,
+      "grad_norm": 0.21521176397800446,
+      "learning_rate": 9.967833782084004e-06,
+      "loss": 0.815,
+      "step": 170
+    },
+    {
+      "epoch": 0.0693107431651906,
+      "grad_norm": 0.22746385633945465,
+      "learning_rate": 9.960385711850554e-06,
+      "loss": 0.8478,
+      "step": 180
+    },
+    {
+      "epoch": 0.07316134000770119,
+      "grad_norm": 0.23073655366897583,
+      "learning_rate": 9.952166110886394e-06,
+      "loss": 0.8346,
+      "step": 190
+    },
+    {
+      "epoch": 0.07701193685021178,
+      "grad_norm": 0.22818072140216827,
+      "learning_rate": 9.943176257655567e-06,
+      "loss": 0.833,
+      "step": 200
+    },
+    {
+      "epoch": 0.08086253369272237,
+      "grad_norm": 0.23068463802337646,
+      "learning_rate": 9.933417550425952e-06,
+      "loss": 0.8187,
+      "step": 210
+    },
+    {
+      "epoch": 0.08471313053523297,
+      "grad_norm": 0.2432355284690857,
+      "learning_rate": 9.9228915070518e-06,
+      "loss": 0.8114,
+      "step": 220
+    },
+    {
+      "epoch": 0.08856372737774355,
+      "grad_norm": 0.21405062079429626,
+      "learning_rate": 9.911599764737632e-06,
+      "loss": 0.8061,
+      "step": 230
+    },
+    {
+      "epoch": 0.09241432422025414,
+      "grad_norm": 0.22993522882461548,
+      "learning_rate": 9.899544079783606e-06,
+      "loss": 0.7997,
+      "step": 240
+    },
+    {
+      "epoch": 0.09626492106276473,
+      "grad_norm": 0.22971560060977936,
+      "learning_rate": 9.886726327312338e-06,
+      "loss": 0.8171,
+      "step": 250
+    },
+    {
+      "epoch": 0.10011551790527531,
+      "grad_norm": 0.20847393572330475,
+      "learning_rate": 9.87314850097724e-06,
+      "loss": 0.8157,
+      "step": 260
+    },
+    {
+      "epoch": 0.1039661147477859,
+      "grad_norm": 0.22684535384178162,
+      "learning_rate": 9.858812712652452e-06,
+      "loss": 0.7966,
+      "step": 270
+    },
+    {
+      "epoch": 0.1078167115902965,
+      "grad_norm": 0.22174301743507385,
+      "learning_rate": 9.843721192104341e-06,
+      "loss": 0.7954,
+      "step": 280
+    },
+    {
+      "epoch": 0.11166730843280709,
+      "grad_norm": 0.20227986574172974,
+      "learning_rate": 9.827876286644698e-06,
+      "loss": 0.8263,
+      "step": 290
+    },
+    {
+      "epoch": 0.11551790527531768,
+      "grad_norm": 0.1897900402545929,
+      "learning_rate": 9.811280460765643e-06,
+      "loss": 0.7845,
+      "step": 300
+    },
+    {
+      "epoch": 0.11936850211782826,
+      "grad_norm": 0.21260590851306915,
+      "learning_rate": 9.793936295756292e-06,
+      "loss": 0.7907,
+      "step": 310
+    },
+    {
+      "epoch": 0.12321909896033885,
+      "grad_norm": 0.207275852560997,
+      "learning_rate": 9.775846489301279e-06,
+      "loss": 0.7988,
+      "step": 320
+    },
+    {
+      "epoch": 0.12706969580284944,
+      "grad_norm": 0.2143951952457428,
+      "learning_rate": 9.757013855061157e-06,
+      "loss": 0.8012,
+      "step": 330
+    },
+    {
+      "epoch": 0.13092029264536004,
+      "grad_norm": 0.20669367909431458,
+      "learning_rate": 9.737441322234758e-06,
+      "loss": 0.8039,
+      "step": 340
+    },
+    {
+      "epoch": 0.1347708894878706,
+      "grad_norm": 0.22080452740192413,
+      "learning_rate": 9.717131935103605e-06,
+      "loss": 0.8103,
+      "step": 350
+    },
+    {
+      "epoch": 0.1386214863303812,
+      "grad_norm": 0.20624369382858276,
+      "learning_rate": 9.696088852558398e-06,
+      "loss": 0.7902,
+      "step": 360
+    },
+    {
+      "epoch": 0.1424720831728918,
+      "grad_norm": 0.2499890923500061,
+      "learning_rate": 9.674315347607686e-06,
+      "loss": 0.8185,
+      "step": 370
+    },
+    {
+      "epoch": 0.14632268001540238,
+      "grad_norm": 0.20117080211639404,
+      "learning_rate": 9.651814806868796e-06,
+      "loss": 0.7835,
+      "step": 380
+    },
+    {
+      "epoch": 0.15017327685791299,
+      "grad_norm": 0.22911939024925232,
+      "learning_rate": 9.62859073004107e-06,
+      "loss": 0.7875,
+      "step": 390
+    },
+    {
+      "epoch": 0.15402387370042356,
+      "grad_norm": 0.1824311763048172,
+      "learning_rate": 9.604646729361547e-06,
+      "loss": 0.7954,
+      "step": 400
+    },
+    {
+      "epoch": 0.15787447054293416,
+      "grad_norm": 0.2137448489665985,
+      "learning_rate": 9.579986529043099e-06,
+      "loss": 0.7922,
+      "step": 410
+    },
+    {
+      "epoch": 0.16172506738544473,
+      "grad_norm": 0.2140379399061203,
+      "learning_rate": 9.554613964695189e-06,
+      "loss": 0.8058,
+      "step": 420
+    },
+    {
+      "epoch": 0.16557566422795533,
+      "grad_norm": 0.204620361328125,
+      "learning_rate": 9.528532982727281e-06,
+      "loss": 0.7848,
+      "step": 430
+    },
+    {
+      "epoch": 0.16942626107046593,
+      "grad_norm": 0.1884375661611557,
+      "learning_rate": 9.501747639735018e-06,
+      "loss": 0.7944,
+      "step": 440
+    },
+    {
+      "epoch": 0.1732768579129765,
+      "grad_norm": 0.20427818596363068,
+      "learning_rate": 9.474262101869272e-06,
+      "loss": 0.7785,
+      "step": 450
+    },
+    {
+      "epoch": 0.1771274547554871,
+      "grad_norm": 0.22005681693553925,
+      "learning_rate": 9.446080644188141e-06,
+      "loss": 0.7875,
+      "step": 460
+    },
+    {
+      "epoch": 0.18097805159799768,
+      "grad_norm": 0.2053992599248886,
+      "learning_rate": 9.417207649992017e-06,
+      "loss": 0.7911,
+      "step": 470
+    },
+    {
+      "epoch": 0.18482864844050828,
+      "grad_norm": 0.2009965479373932,
+      "learning_rate": 9.387647610141816e-06,
+      "loss": 0.7913,
+      "step": 480
+    },
+    {
+      "epoch": 0.18867924528301888,
+      "grad_norm": 0.18809419870376587,
+      "learning_rate": 9.35740512236047e-06,
+      "loss": 0.7831,
+      "step": 490
+    },
+    {
+      "epoch": 0.19252984212552945,
+      "grad_norm": 0.18447789549827576,
+      "learning_rate": 9.326484890517809e-06,
+      "loss": 0.785,
+      "step": 500
+    },
+    {
+      "epoch": 0.19638043896804006,
+      "grad_norm": 0.20674534142017365,
+      "learning_rate": 9.294891723898926e-06,
+      "loss": 0.7764,
+      "step": 510
+    },
+    {
+      "epoch": 0.20023103581055063,
+      "grad_norm": 0.18783864378929138,
+      "learning_rate": 9.262630536456153e-06,
+      "loss": 0.8011,
+      "step": 520
+    },
+    {
+      "epoch": 0.20408163265306123,
+      "grad_norm": 0.20276913046836853,
+      "learning_rate": 9.229706346044749e-06,
+      "loss": 0.7835,
+      "step": 530
+    },
+    {
+      "epoch": 0.2079322294955718,
+      "grad_norm": 0.20508168637752533,
+      "learning_rate": 9.196124273642422e-06,
+      "loss": 0.793,
+      "step": 540
+    },
+    {
+      "epoch": 0.2117828263380824,
+      "grad_norm": 0.1926833838224411,
+      "learning_rate": 9.161889542552842e-06,
+      "loss": 0.7977,
+      "step": 550
+    },
+    {
+      "epoch": 0.215633423180593,
+      "grad_norm": 0.21568267047405243,
+      "learning_rate": 9.127007477593193e-06,
+      "loss": 0.8017,
+      "step": 560
+    },
+    {
+      "epoch": 0.21948402002310358,
+      "grad_norm": 0.22079801559448242,
+      "learning_rate": 9.091483504265974e-06,
+      "loss": 0.8193,
+      "step": 570
+    },
+    {
+      "epoch": 0.22333461686561418,
+      "grad_norm": 0.20382393896579742,
+      "learning_rate": 9.055323147915116e-06,
+      "loss": 0.7824,
+      "step": 580
+    },
+    {
+      "epoch": 0.22718521370812475,
+      "grad_norm": 0.1937755048274994,
+      "learning_rate": 9.018532032866585e-06,
+      "loss": 0.8004,
+      "step": 590
+    },
+    {
+      "epoch": 0.23103581055063535,
+      "grad_norm": 0.1790451854467392,
+      "learning_rate": 8.981115881553579e-06,
+      "loss": 0.7975,
+      "step": 600
+    },
+    {
+      "epoch": 0.23488640739314592,
+      "grad_norm": 0.20686203241348267,
+      "learning_rate": 8.943080513626476e-06,
+      "loss": 0.8153,
+      "step": 610
+    },
+    {
+      "epoch": 0.23873700423565652,
+      "grad_norm": 0.18961578607559204,
+      "learning_rate": 8.904431845047641e-06,
+      "loss": 0.7927,
+      "step": 620
+    },
+    {
+      "epoch": 0.24258760107816713,
+      "grad_norm": 0.2054220736026764,
+      "learning_rate": 8.86517588717128e-06,
+      "loss": 0.8051,
+      "step": 630
+    },
+    {
+      "epoch": 0.2464381979206777,
+      "grad_norm": 0.23013338446617126,
+      "learning_rate": 8.82531874580844e-06,
+      "loss": 0.8003,
+      "step": 640
+    },
+    {
+      "epoch": 0.25028879476318827,
+      "grad_norm": 0.1904127150774002,
+      "learning_rate": 8.784866620277314e-06,
+      "loss": 0.762,
+      "step": 650
+    },
+    {
+      "epoch": 0.25413939160569887,
+      "grad_norm": 0.21796554327011108,
+      "learning_rate": 8.743825802439018e-06,
+      "loss": 0.7907,
+      "step": 660
+    },
+    {
+      "epoch": 0.2579899884482095,
+      "grad_norm": 0.19501149654388428,
+      "learning_rate": 8.702202675718956e-06,
+      "loss": 0.7966,
+      "step": 670
+    },
+    {
+      "epoch": 0.2618405852907201,
+      "grad_norm": 0.18285229802131653,
+      "learning_rate": 8.660003714113959e-06,
+      "loss": 0.7802,
+      "step": 680
+    },
+    {
+      "epoch": 0.2656911821332307,
+      "grad_norm": 0.1969471573829651,
+      "learning_rate": 8.617235481185323e-06,
+      "loss": 0.7919,
+      "step": 690
+    },
+    {
+      "epoch": 0.2695417789757412,
+      "grad_norm": 0.19401787221431732,
+      "learning_rate": 8.573904629037931e-06,
+      "loss": 0.8019,
+      "step": 700
+    },
+    {
+      "epoch": 0.2733923758182518,
+      "grad_norm": 0.1793961077928543,
+      "learning_rate": 8.530017897285585e-06,
+      "loss": 0.782,
+      "step": 710
+    },
+    {
+      "epoch": 0.2772429726607624,
+      "grad_norm": 0.20331700146198273,
+      "learning_rate": 8.485582112002746e-06,
+      "loss": 0.7842,
+      "step": 720
+    },
+    {
+      "epoch": 0.281093569503273,
+      "grad_norm": 0.21211951971054077,
+      "learning_rate": 8.440604184662811e-06,
+      "loss": 0.7948,
+      "step": 730
+    },
+    {
+      "epoch": 0.2849441663457836,
+      "grad_norm": 0.2009010761976242,
+      "learning_rate": 8.395091111063118e-06,
+      "loss": 0.7808,
+      "step": 740
+    },
+    {
+      "epoch": 0.28879476318829417,
+      "grad_norm": 0.17986911535263062,
+      "learning_rate": 8.349049970236822e-06,
+      "loss": 0.7753,
+      "step": 750
+    },
+    {
+      "epoch": 0.29264536003080477,
+      "grad_norm": 0.19532844424247742,
+      "learning_rate": 8.302487923351842e-06,
+      "loss": 0.7737,
+      "step": 760
+    },
+    {
+      "epoch": 0.29649595687331537,
+      "grad_norm": 0.1870134174823761,
+      "learning_rate": 8.255412212597021e-06,
+      "loss": 0.7925,
+      "step": 770
+    },
+    {
+      "epoch": 0.30034655371582597,
+      "grad_norm": 0.19853828847408295,
+      "learning_rate": 8.207830160055688e-06,
+      "loss": 0.7865,
+      "step": 780
+    },
+    {
+      "epoch": 0.3041971505583365,
+      "grad_norm": 0.1867004930973053,
+      "learning_rate": 8.159749166566786e-06,
+      "loss": 0.772,
+      "step": 790
+    },
+    {
+      "epoch": 0.3080477474008471,
+      "grad_norm": 0.1964997947216034,
+      "learning_rate": 8.11117671057377e-06,
+      "loss": 0.7934,
+      "step": 800
+    },
+    {
+      "epoch": 0.3118983442433577,
+      "grad_norm": 0.1931861788034439,
+      "learning_rate": 8.062120346961404e-06,
+      "loss": 0.7923,
+      "step": 810
+    },
+    {
+      "epoch": 0.3157489410858683,
+      "grad_norm": 0.18405266106128693,
+      "learning_rate": 8.012587705880705e-06,
+      "loss": 0.7899,
+      "step": 820
+    },
+    {
+      "epoch": 0.3195995379283789,
+      "grad_norm": 0.19046756625175476,
+      "learning_rate": 7.962586491562146e-06,
+      "loss": 0.7758,
+      "step": 830
+    },
+    {
+      "epoch": 0.32345013477088946,
+      "grad_norm": 0.20039959251880646,
+      "learning_rate": 7.912124481117357e-06,
+      "loss": 0.7744,
+      "step": 840
+    },
+    {
+      "epoch": 0.32730073161340006,
+      "grad_norm": 0.1728489249944687,
+      "learning_rate": 7.861209523329494e-06,
+      "loss": 0.7837,
+      "step": 850
+    },
+    {
+      "epoch": 0.33115132845591067,
+      "grad_norm": 0.19496537744998932,
+      "learning_rate": 7.809849537432432e-06,
+      "loss": 0.7779,
+      "step": 860
+    },
+    {
+      "epoch": 0.33500192529842127,
+      "grad_norm": 0.1852690428495407,
+      "learning_rate": 7.758052511879035e-06,
+      "loss": 0.7795,
+      "step": 870
+    },
+    {
+      "epoch": 0.33885252214093187,
+      "grad_norm": 0.2045857310295105,
+      "learning_rate": 7.705826503098642e-06,
+      "loss": 0.7828,
+      "step": 880
+    },
+    {
+      "epoch": 0.3427031189834424,
+      "grad_norm": 0.19683398306369781,
+      "learning_rate": 7.653179634243979e-06,
+      "loss": 0.775,
+      "step": 890
+    },
+    {
+      "epoch": 0.346553715825953,
+      "grad_norm": 0.18332123756408691,
+      "learning_rate": 7.600120093927696e-06,
+      "loss": 0.7804,
+      "step": 900
+    },
+    {
+      "epoch": 0.3504043126684636,
+      "grad_norm": 0.20081990957260132,
+      "learning_rate": 7.546656134948729e-06,
+      "loss": 0.7701,
+      "step": 910
+    },
+    {
+      "epoch": 0.3542549095109742,
+      "grad_norm": 0.1897205114364624,
+      "learning_rate": 7.492796073008661e-06,
+      "loss": 0.7748,
+      "step": 920
+    },
+    {
+      "epoch": 0.3581055063534848,
+      "grad_norm": 0.2149105817079544,
+      "learning_rate": 7.438548285418326e-06,
+      "loss": 0.7573,
+      "step": 930
+    },
+    {
+      "epoch": 0.36195610319599536,
+      "grad_norm": 0.1990543156862259,
+      "learning_rate": 7.3839212097948e-06,
+      "loss": 0.7877,
+      "step": 940
+    },
+    {
+      "epoch": 0.36580670003850596,
+      "grad_norm": 0.20030899345874786,
+      "learning_rate": 7.328923342749039e-06,
+      "loss": 0.8047,
+      "step": 950
+    },
+    {
+      "epoch": 0.36965729688101656,
+      "grad_norm": 0.18734586238861084,
+      "learning_rate": 7.273563238564319e-06,
+      "loss": 0.7996,
+      "step": 960
+    },
+    {
+      "epoch": 0.37350789372352716,
+      "grad_norm": 0.1962006390094757,
+      "learning_rate": 7.217849507865724e-06,
+      "loss": 0.7707,
+      "step": 970
+    },
+    {
+      "epoch": 0.37735849056603776,
+      "grad_norm": 0.1857118010520935,
+      "learning_rate": 7.161790816280858e-06,
+      "loss": 0.7769,
+      "step": 980
+    },
+    {
+      "epoch": 0.3812090874085483,
+      "grad_norm": 0.19783788919448853,
+      "learning_rate": 7.1053958830920125e-06,
+      "loss": 0.7681,
+      "step": 990
+    },
+    {
+      "epoch": 0.3850596842510589,
+      "grad_norm": 0.1971602439880371,
+      "learning_rate": 7.0486734798799715e-06,
+      "loss": 0.776,
+      "step": 1000
+    },
+    {
+      "epoch": 0.3889102810935695,
+      "grad_norm": 0.18284115195274353,
+      "learning_rate": 6.991632429159708e-06,
+      "loss": 0.7951,
+      "step": 1010
+    },
+    {
+      "epoch": 0.3927608779360801,
+      "grad_norm": 0.20390371978282928,
+      "learning_rate": 6.934281603008132e-06,
+      "loss": 0.7958,
+      "step": 1020
+    },
+    {
+      "epoch": 0.39661147477859066,
+      "grad_norm": 0.17112939059734344,
+      "learning_rate": 6.876629921684143e-06,
+      "loss": 0.7924,
+      "step": 1030
+    },
+    {
+      "epoch": 0.40046207162110126,
+      "grad_norm": 0.19549788534641266,
+      "learning_rate": 6.818686352241199e-06,
+      "loss": 0.7805,
+      "step": 1040
+    },
+    {
+      "epoch": 0.40431266846361186,
+      "grad_norm": 0.17474836111068726,
+      "learning_rate": 6.760459907132573e-06,
+      "loss": 0.7769,
+      "step": 1050
+    },
+    {
+      "epoch": 0.40816326530612246,
+      "grad_norm": 0.17632028460502625,
+      "learning_rate": 6.701959642809589e-06,
+      "loss": 0.7822,
+      "step": 1060
+    },
+    {
+      "epoch": 0.41201386214863306,
+      "grad_norm": 0.1951586902141571,
+      "learning_rate": 6.643194658312987e-06,
+      "loss": 0.7743,
+      "step": 1070
+    },
+    {
+      "epoch": 0.4158644589911436,
+      "grad_norm": 0.1756243258714676,
+      "learning_rate": 6.584174093857676e-06,
+      "loss": 0.799,
+      "step": 1080
+    },
+    {
+      "epoch": 0.4197150558336542,
+      "grad_norm": 0.20384393632411957,
+      "learning_rate": 6.5249071294110735e-06,
+      "loss": 0.7808,
+      "step": 1090
+    },
+    {
+      "epoch": 0.4235656526761648,
+      "grad_norm": 0.20352546870708466,
+      "learning_rate": 6.465402983265274e-06,
+      "loss": 0.7928,
+      "step": 1100
+    },
+    {
+      "epoch": 0.4274162495186754,
+      "grad_norm": 0.20659860968589783,
+      "learning_rate": 6.405670910603243e-06,
+      "loss": 0.7871,
+      "step": 1110
+    },
+    {
+      "epoch": 0.431266846361186,
+      "grad_norm": 0.20375967025756836,
+      "learning_rate": 6.345720202059291e-06,
+      "loss": 0.7846,
+      "step": 1120
+    },
+    {
+      "epoch": 0.43511744320369655,
+      "grad_norm": 0.2004343569278717,
+      "learning_rate": 6.285560182274008e-06,
+      "loss": 0.7649,
+      "step": 1130
+    },
+    {
+      "epoch": 0.43896804004620715,
+      "grad_norm": 0.1995244324207306,
+      "learning_rate": 6.225200208443936e-06,
+      "loss": 0.7888,
+      "step": 1140
+    },
+    {
+      "epoch": 0.44281863688871775,
+      "grad_norm": 0.18316124379634857,
+      "learning_rate": 6.16464966886616e-06,
+      "loss": 0.7744,
+      "step": 1150
+    },
+    {
+      "epoch": 0.44666923373122835,
+      "grad_norm": 0.22643060982227325,
+      "learning_rate": 6.103917981478065e-06,
+      "loss": 0.8023,
+      "step": 1160
+    },
+    {
+      "epoch": 0.45051983057373896,
+      "grad_norm": 0.17931228876113892,
+      "learning_rate": 6.043014592392489e-06,
+      "loss": 0.81,
+      "step": 1170
+    },
+    {
+      "epoch": 0.4543704274162495,
+      "grad_norm": 0.18572434782981873,
+      "learning_rate": 5.981948974428485e-06,
+      "loss": 0.7887,
+      "step": 1180
+    },
+    {
+      "epoch": 0.4582210242587601,
+      "grad_norm": 0.18428096175193787,
+      "learning_rate": 5.920730625637934e-06,
+      "loss": 0.7682,
+      "step": 1190
+    },
+    {
+      "epoch": 0.4620716211012707,
+      "grad_norm": 0.1926165521144867,
+      "learning_rate": 5.8593690678282355e-06,
+      "loss": 0.7739,
+      "step": 1200
+    },
+    {
+      "epoch": 0.4659222179437813,
+      "grad_norm": 0.1930193454027176,
+      "learning_rate": 5.797873845081301e-06,
+      "loss": 0.7727,
+      "step": 1210
+    },
+    {
+      "epoch": 0.46977281478629185,
+      "grad_norm": 0.1982874572277069,
+      "learning_rate": 5.736254522269075e-06,
+      "loss": 0.7987,
+      "step": 1220
+    },
+    {
+      "epoch": 0.47362341162880245,
+      "grad_norm": 0.18213649094104767,
+      "learning_rate": 5.6745206835658386e-06,
+      "loss": 0.7928,
+      "step": 1230
+    },
+    {
+      "epoch": 0.47747400847131305,
+      "grad_norm": 0.19462884962558746,
+      "learning_rate": 5.612681930957495e-06,
+      "loss": 0.7719,
+      "step": 1240
+    },
+    {
+      "epoch": 0.48132460531382365,
+      "grad_norm": 0.18728210031986237,
+      "learning_rate": 5.550747882748095e-06,
+      "loss": 0.7661,
+      "step": 1250
+    },
+    {
+      "epoch": 0.48517520215633425,
+      "grad_norm": 0.22729694843292236,
+      "learning_rate": 5.4887281720638195e-06,
+      "loss": 0.7778,
+      "step": 1260
+    },
+    {
+      "epoch": 0.4890257989988448,
+      "grad_norm": 0.2072780728340149,
+      "learning_rate": 5.42663244535466e-06,
+      "loss": 0.758,
+      "step": 1270
+    },
+    {
+      "epoch": 0.4928763958413554,
+      "grad_norm": 0.19957545399665833,
+      "learning_rate": 5.3644703608940205e-06,
+      "loss": 0.7715,
+      "step": 1280
+    },
+    {
+      "epoch": 0.496726992683866,
+      "grad_norm": 0.17197726666927338,
+      "learning_rate": 5.302251587276491e-06,
+      "loss": 0.7845,
+      "step": 1290
+    },
+    {
+      "epoch": 0.5005775895263765,
+      "grad_norm": 0.19657622277736664,
+      "learning_rate": 5.2399858019140005e-06,
+      "loss": 0.7747,
+      "step": 1300
+    },
+    {
+      "epoch": 0.5044281863688872,
+      "grad_norm": 0.18291780352592468,
+      "learning_rate": 5.177682689530615e-06,
+      "loss": 0.7914,
+      "step": 1310
+    },
+    {
+      "epoch": 0.5082787832113977,
+      "grad_norm": 0.20224712789058685,
+      "learning_rate": 5.11535194065619e-06,
+      "loss": 0.7653,
+      "step": 1320
+    },
+    {
+      "epoch": 0.5121293800539084,
+      "grad_norm": 0.1915288120508194,
+      "learning_rate": 5.053003250119113e-06,
+      "loss": 0.7746,
+      "step": 1330
+    },
+    {
+      "epoch": 0.515979976896419,
+      "grad_norm": 0.20163559913635254,
+      "learning_rate": 4.9906463155383925e-06,
+      "loss": 0.7786,
+      "step": 1340
+    },
+    {
+      "epoch": 0.5198305737389295,
+      "grad_norm": 0.21704857051372528,
+      "learning_rate": 4.9282908358153046e-06,
+      "loss": 0.7741,
+      "step": 1350
+    },
+    {
+      "epoch": 0.5236811705814401,
+      "grad_norm": 0.20458900928497314,
+      "learning_rate": 4.8659465096248345e-06,
+      "loss": 0.792,
+      "step": 1360
+    },
+    {
+      "epoch": 0.5275317674239507,
+      "grad_norm": 0.1971500962972641,
+      "learning_rate": 4.803623033907169e-06,
+      "loss": 0.7862,
+      "step": 1370
+    },
+    {
+      "epoch": 0.5313823642664613,
+      "grad_norm": 0.18453796207904816,
+      "learning_rate": 4.741330102359441e-06,
+      "loss": 0.7973,
+      "step": 1380
+    },
+    {
+      "epoch": 0.5352329611089719,
+      "grad_norm": 0.2027163952589035,
+      "learning_rate": 4.679077403927996e-06,
+      "loss": 0.774,
+      "step": 1390
+    },
+    {
+      "epoch": 0.5390835579514824,
+      "grad_norm": 0.19793139398097992,
+      "learning_rate": 4.616874621301373e-06,
+      "loss": 0.7743,
+      "step": 1400
+    },
+    {
+      "epoch": 0.5429341547939931,
+      "grad_norm": 0.202447772026062,
+      "learning_rate": 4.554731429404293e-06,
+      "loss": 0.7911,
+      "step": 1410
+    },
+    {
+      "epoch": 0.5467847516365036,
+      "grad_norm": 0.1947924792766571,
+      "learning_rate": 4.492657493892822e-06,
+      "loss": 0.7771,
+      "step": 1420
+    },
+    {
+      "epoch": 0.5506353484790143,
+      "grad_norm": 0.19378690421581268,
+      "learning_rate": 4.430662469650999e-06,
+      "loss": 0.7892,
+      "step": 1430
+    },
+    {
+      "epoch": 0.5544859453215248,
+      "grad_norm": 0.19362278282642365,
+      "learning_rate": 4.368755999289118e-06,
+      "loss": 0.7878,
+      "step": 1440
+    },
+    {
+      "epoch": 0.5583365421640354,
+      "grad_norm": 0.19865773618221283,
+      "learning_rate": 4.306947711643949e-06,
+      "loss": 0.7732,
+      "step": 1450
+    },
+    {
+      "epoch": 0.562187139006546,
+      "grad_norm": 0.19222889840602875,
+      "learning_rate": 4.245247220281068e-06,
+      "loss": 0.7774,
+      "step": 1460
+    },
+    {
+      "epoch": 0.5660377358490566,
+      "grad_norm": 0.18325470387935638,
+      "learning_rate": 4.183664121999587e-06,
+      "loss": 0.7757,
+      "step": 1470
+    },
+    {
+      "epoch": 0.5698883326915672,
+      "grad_norm": 0.1971128284931183,
+      "learning_rate": 4.122207995339474e-06,
+      "loss": 0.7672,
+      "step": 1480
+    },
+    {
+      "epoch": 0.5737389295340778,
+      "grad_norm": 0.18651960790157318,
+      "learning_rate": 4.060888399091734e-06,
+      "loss": 0.777,
+      "step": 1490
+    },
+    {
+      "epoch": 0.5775895263765883,
+      "grad_norm": 0.18244920670986176,
+      "learning_rate": 3.999714870811639e-06,
+      "loss": 0.787,
+      "step": 1500
+    },
+    {
+      "epoch": 0.581440123219099,
+      "grad_norm": 0.21237461268901825,
+      "learning_rate": 3.938696925335275e-06,
+      "loss": 0.782,
+      "step": 1510
+    },
+    {
+      "epoch": 0.5852907200616095,
+      "grad_norm": 0.18905198574066162,
+      "learning_rate": 3.87784405329962e-06,
+      "loss": 0.7896,
+      "step": 1520
+    },
+    {
+      "epoch": 0.5891413169041202,
+      "grad_norm": 0.18127086758613586,
+      "learning_rate": 3.817165719666382e-06,
+      "loss": 0.792,
+      "step": 1530
+    },
+    {
+      "epoch": 0.5929919137466307,
+      "grad_norm": 0.18326129019260406,
+      "learning_rate": 3.756671362249841e-06,
+      "loss": 0.7898,
+      "step": 1540
+    },
+    {
+      "epoch": 0.5968425105891413,
+      "grad_norm": 0.18368223309516907,
+      "learning_rate": 3.6963703902488907e-06,
+      "loss": 0.779,
+      "step": 1550
+    },
+    {
+      "epoch": 0.6006931074316519,
+      "grad_norm": 0.1841137558221817,
+      "learning_rate": 3.6362721827835655e-06,
+      "loss": 0.794,
+      "step": 1560
+    },
+    {
+      "epoch": 0.6045437042741625,
+      "grad_norm": 0.19373591244220734,
+      "learning_rate": 3.576386087436212e-06,
+      "loss": 0.7952,
+      "step": 1570
+    },
+    {
+      "epoch": 0.608394301116673,
+      "grad_norm": 0.20081838965415955,
+      "learning_rate": 3.5167214187975913e-06,
+      "loss": 0.7888,
+      "step": 1580
+    },
+    {
+      "epoch": 0.6122448979591837,
+      "grad_norm": 0.1750497668981552,
+      "learning_rate": 3.457287457018086e-06,
+      "loss": 0.7838,
+      "step": 1590
+    },
+    {
+      "epoch": 0.6160954948016942,
+      "grad_norm": 0.193156436085701,
+      "learning_rate": 3.3980934463643003e-06,
+      "loss": 0.7797,
+      "step": 1600
+    },
+    {
+      "epoch": 0.6199460916442049,
+      "grad_norm": 0.17371925711631775,
+      "learning_rate": 3.3391485937812053e-06,
+      "loss": 0.7573,
+      "step": 1610
+    },
+    {
+      "epoch": 0.6237966884867154,
+      "grad_norm": 0.21084265410900116,
+      "learning_rate": 3.280462067460115e-06,
+      "loss": 0.7588,
+      "step": 1620
+    },
+    {
+      "epoch": 0.627647285329226,
+      "grad_norm": 0.18486298620700836,
+      "learning_rate": 3.222042995412669e-06,
+      "loss": 0.7836,
+      "step": 1630
+    },
+    {
+      "epoch": 0.6314978821717366,
+      "grad_norm": 0.16794146597385406,
+      "learning_rate": 3.163900464051089e-06,
+      "loss": 0.7751,
+      "step": 1640
+    },
+    {
+      "epoch": 0.6353484790142472,
+      "grad_norm": 0.1916816085577011,
+      "learning_rate": 3.1060435167748826e-06,
+      "loss": 0.7891,
+      "step": 1650
+    },
+    {
+      "epoch": 0.6391990758567578,
+      "grad_norm": 0.18842566013336182,
+      "learning_rate": 3.048481152564253e-06,
+      "loss": 0.7652,
+      "step": 1660
+    },
+    {
+      "epoch": 0.6430496726992684,
+      "grad_norm": 0.19945907592773438,
+      "learning_rate": 2.9912223245804017e-06,
+      "loss": 0.7829,
+      "step": 1670
+    },
+    {
+      "epoch": 0.6469002695417789,
+      "grad_norm": 0.18741340935230255,
+      "learning_rate": 2.934275938772981e-06,
+      "loss": 0.7777,
+      "step": 1680
+    },
+    {
+      "epoch": 0.6507508663842896,
+      "grad_norm": 0.20435859262943268,
+      "learning_rate": 2.8776508524948676e-06,
+      "loss": 0.7718,
+      "step": 1690
+    },
+    {
+      "epoch": 0.6546014632268001,
+      "grad_norm": 0.18108192086219788,
+      "learning_rate": 2.821355873124503e-06,
+      "loss": 0.7758,
+      "step": 1700
+    },
+    {
+      "epoch": 0.6584520600693108,
+      "grad_norm": 0.2008640170097351,
+      "learning_rate": 2.7653997566960144e-06,
+      "loss": 0.7788,
+      "step": 1710
+    },
+    {
+      "epoch": 0.6623026569118213,
+      "grad_norm": 0.20901796221733093,
+      "learning_rate": 2.709791206537309e-06,
+      "loss": 0.7723,
+      "step": 1720
+    },
+    {
+      "epoch": 0.6661532537543319,
+      "grad_norm": 0.20172666013240814,
+      "learning_rate": 2.6545388719163747e-06,
+      "loss": 0.7808,
+      "step": 1730
+    },
+    {
+      "epoch": 0.6700038505968425,
+      "grad_norm": 0.19516818225383759,
+      "learning_rate": 2.599651346695979e-06,
+      "loss": 0.7887,
+      "step": 1740
+    },
+    {
+      "epoch": 0.6738544474393531,
+      "grad_norm": 0.17920154333114624,
+      "learning_rate": 2.545137167997007e-06,
+      "loss": 0.7791,
+      "step": 1750
+    },
+    {
+      "epoch": 0.6777050442818637,
+      "grad_norm": 0.1931513100862503,
+      "learning_rate": 2.4910048148705972e-06,
+      "loss": 0.7752,
+      "step": 1760
+    },
+    {
+      "epoch": 0.6815556411243743,
+      "grad_norm": 0.19972948729991913,
+      "learning_rate": 2.4372627069793327e-06,
+      "loss": 0.8082,
+      "step": 1770
+    },
+    {
+      "epoch": 0.6854062379668848,
+      "grad_norm": 0.18122310936450958,
+      "learning_rate": 2.3839192032876486e-06,
+      "loss": 0.7867,
+      "step": 1780
+    },
+    {
+      "epoch": 0.6892568348093955,
+      "grad_norm": 0.19373857975006104,
+      "learning_rate": 2.3309826007617047e-06,
+      "loss": 0.8017,
+      "step": 1790
+    },
+    {
+      "epoch": 0.693107431651906,
+      "grad_norm": 0.18891261518001556,
+      "learning_rate": 2.278461133078881e-06,
+      "loss": 0.7687,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6969580284944167,
+      "grad_norm": 0.2011217325925827,
+      "learning_rate": 2.2263629693471258e-06,
+      "loss": 0.7635,
+      "step": 1810
+    },
+    {
+      "epoch": 0.7008086253369272,
+      "grad_norm": 0.18512079119682312,
+      "learning_rate": 2.1746962128343414e-06,
+      "loss": 0.7878,
+      "step": 1820
+    },
+    {
+      "epoch": 0.7046592221794378,
+      "grad_norm": 0.18732482194900513,
+      "learning_rate": 2.1234688997080183e-06,
+      "loss": 0.7694,
+      "step": 1830
+    },
+    {
+      "epoch": 0.7085098190219484,
+      "grad_norm": 0.18888023495674133,
+      "learning_rate": 2.0726889977853053e-06,
+      "loss": 0.7741,
+      "step": 1840
+    },
+    {
+      "epoch": 0.712360415864459,
+      "grad_norm": 0.19380293786525726,
+      "learning_rate": 2.022364405293703e-06,
+      "loss": 0.7882,
+      "step": 1850
+    },
+    {
+      "epoch": 0.7162110127069696,
+      "grad_norm": 0.19045989215373993,
+      "learning_rate": 1.9725029496425814e-06,
+      "loss": 0.7867,
+      "step": 1860
+    },
+    {
+      "epoch": 0.7200616095494802,
+      "grad_norm": 0.1907743364572525,
+      "learning_rate": 1.923112386205727e-06,
+      "loss": 0.7776,
+      "step": 1870
+    },
+    {
+      "epoch": 0.7239122063919907,
+      "grad_norm": 0.18936869502067566,
+      "learning_rate": 1.874200397115079e-06,
+      "loss": 0.7888,
+      "step": 1880
+    },
+    {
+      "epoch": 0.7277628032345014,
+      "grad_norm": 0.19421523809432983,
+      "learning_rate": 1.825774590065857e-06,
+      "loss": 0.775,
+      "step": 1890
+    },
+    {
+      "epoch": 0.7316134000770119,
+      "grad_norm": 0.2049892246723175,
+      "learning_rate": 1.7778424971332813e-06,
+      "loss": 0.7851,
+      "step": 1900
+    },
+    {
+      "epoch": 0.7354639969195226,
+      "grad_norm": 0.17711523175239563,
+      "learning_rate": 1.7304115736010407e-06,
+      "loss": 0.7847,
+      "step": 1910
+    },
+    {
+      "epoch": 0.7393145937620331,
+      "grad_norm": 0.1924167424440384,
+      "learning_rate": 1.6834891968017108e-06,
+      "loss": 0.7706,
+      "step": 1920
+    },
+    {
+      "epoch": 0.7431651906045437,
+      "grad_norm": 0.2022978961467743,
+      "learning_rate": 1.6370826649692884e-06,
+      "loss": 0.7927,
+      "step": 1930
+    },
+    {
+      "epoch": 0.7470157874470543,
+      "grad_norm": 0.18197661638259888,
+      "learning_rate": 1.5911991961040463e-06,
+      "loss": 0.7887,
+      "step": 1940
+    },
+    {
+      "epoch": 0.7508663842895649,
+      "grad_norm": 0.19015692174434662,
+      "learning_rate": 1.545845926849851e-06,
+      "loss": 0.7695,
+      "step": 1950
+    },
+    {
+      "epoch": 0.7547169811320755,
+      "grad_norm": 0.2005390077829361,
+      "learning_rate": 1.5010299113841397e-06,
+      "loss": 0.7982,
+      "step": 1960
+    },
+    {
+      "epoch": 0.7585675779745861,
+      "grad_norm": 0.19571354985237122,
+      "learning_rate": 1.4567581203207238e-06,
+      "loss": 0.7858,
+      "step": 1970
+    },
+    {
+      "epoch": 0.7624181748170966,
+      "grad_norm": 0.20352637767791748,
+      "learning_rate": 1.413037439625593e-06,
+      "loss": 0.7861,
+      "step": 1980
+    },
+    {
+      "epoch": 0.7662687716596073,
+      "grad_norm": 0.21648868918418884,
+      "learning_rate": 1.3698746695458847e-06,
+      "loss": 0.7726,
+      "step": 1990
+    },
+    {
+      "epoch": 0.7701193685021178,
+      "grad_norm": 0.17887210845947266,
+      "learning_rate": 1.3272765235521745e-06,
+      "loss": 0.7752,
+      "step": 2000
+    },
+    {
+      "epoch": 0.7739699653446284,
+      "grad_norm": 0.18643644452095032,
+      "learning_rate": 1.2852496272942877e-06,
+      "loss": 0.7809,
+      "step": 2010
+    },
+    {
+      "epoch": 0.777820562187139,
+      "grad_norm": 0.19412319362163544,
+      "learning_rate": 1.2438005175707396e-06,
+      "loss": 0.7639,
+      "step": 2020
+    },
+    {
+      "epoch": 0.7816711590296496,
+      "grad_norm": 0.19017969071865082,
+      "learning_rate": 1.2029356413120297e-06,
+      "loss": 0.7609,
+      "step": 2030
+    },
+    {
+      "epoch": 0.7855217558721602,
+      "grad_norm": 0.19313178956508636,
+      "learning_rate": 1.1626613545778786e-06,
+      "loss": 0.7752,
+      "step": 2040
+    },
+    {
+      "epoch": 0.7893723527146708,
+      "grad_norm": 0.18113933503627777,
+      "learning_rate": 1.1229839215686294e-06,
+      "loss": 0.7772,
+      "step": 2050
+    },
+    {
+      "epoch": 0.7932229495571813,
+      "grad_norm": 0.1752539873123169,
+      "learning_rate": 1.0839095136509188e-06,
+      "loss": 0.7643,
+      "step": 2060
+    },
+    {
+      "epoch": 0.797073546399692,
+      "grad_norm": 0.20080353319644928,
+      "learning_rate": 1.045444208397791e-06,
+      "loss": 0.7818,
+      "step": 2070
+    },
+    {
+      "epoch": 0.8009241432422025,
+      "grad_norm": 0.17194822430610657,
+      "learning_rate": 1.007593988643402e-06,
+      "loss": 0.7558,
+      "step": 2080
+    },
+    {
+      "epoch": 0.8047747400847132,
+      "grad_norm": 0.1814519315958023,
+      "learning_rate": 9.703647415524613e-07,
+      "loss": 0.7822,
+      "step": 2090
+    },
+    {
+      "epoch": 0.8086253369272237,
+      "grad_norm": 0.18271952867507935,
+      "learning_rate": 9.337622577045485e-07,
+      "loss": 0.7765,
+      "step": 2100
+    },
+    {
+      "epoch": 0.8124759337697343,
+      "grad_norm": 0.18020667135715485,
+      "learning_rate": 8.977922301934588e-07,
+      "loss": 0.7767,
+      "step": 2110
+    },
+    {
+      "epoch": 0.8163265306122449,
+      "grad_norm": 0.2101748138666153,
+      "learning_rate": 8.624602537417009e-07,
+      "loss": 0.7906,
+      "step": 2120
+    },
+    {
+      "epoch": 0.8201771274547555,
+      "grad_norm": 0.20148402452468872,
+      "learning_rate": 8.277718238303123e-07,
+      "loss": 0.7736,
+      "step": 2130
+    },
+    {
+      "epoch": 0.8240277242972661,
+      "grad_norm": 0.18686886131763458,
+      "learning_rate": 7.937323358440935e-07,
+      "loss": 0.7753,
+      "step": 2140
+    },
+    {
+      "epoch": 0.8278783211397767,
+      "grad_norm": 0.185373455286026,
+      "learning_rate": 7.603470842324223e-07,
+      "loss": 0.7658,
+      "step": 2150
+    },
+    {
+      "epoch": 0.8317289179822872,
+      "grad_norm": 0.18153372406959534,
+      "learning_rate": 7.276212616857575e-07,
+      "loss": 0.7856,
+      "step": 2160
+    },
+    {
+      "epoch": 0.8355795148247979,
+      "grad_norm": 0.1936536729335785,
+      "learning_rate": 6.955599583279837e-07,
+      "loss": 0.7805,
+      "step": 2170
+    },
+    {
+      "epoch": 0.8394301116673084,
+      "grad_norm": 0.17915000021457672,
+      "learning_rate": 6.641681609246981e-07,
+      "loss": 0.7737,
+      "step": 2180
+    },
+    {
+      "epoch": 0.8432807085098191,
+      "grad_norm": 0.19518205523490906,
+      "learning_rate": 6.334507521075711e-07,
+      "loss": 0.7772,
+      "step": 2190
+    },
+    {
+      "epoch": 0.8471313053523296,
+      "grad_norm": 0.20079658925533295,
+      "learning_rate": 6.034125096149179e-07,
+      "loss": 0.7777,
+      "step": 2200
+    },
+    {
+      "epoch": 0.8509819021948402,
+      "grad_norm": 0.19285249710083008,
+      "learning_rate": 5.740581055485722e-07,
+      "loss": 0.7494,
+      "step": 2210
+    },
+    {
+      "epoch": 0.8548324990373508,
+      "grad_norm": 0.1784435510635376,
+      "learning_rate": 5.453921056471961e-07,
+      "loss": 0.783,
+      "step": 2220
+    },
+    {
+      "epoch": 0.8586830958798614,
+      "grad_norm": 0.20434586703777313,
+      "learning_rate": 5.174189685761288e-07,
+      "loss": 0.7764,
+      "step": 2230
+    },
+    {
+      "epoch": 0.862533692722372,
+      "grad_norm": 0.17884820699691772,
+      "learning_rate": 4.901430452338979e-07,
+      "loss": 0.7857,
+      "step": 2240
+    },
+    {
+      "epoch": 0.8663842895648826,
+      "grad_norm": 0.1851227730512619,
+      "learning_rate": 4.6356857807548286e-07,
+      "loss": 0.7757,
+      "step": 2250
+    },
+    {
+      "epoch": 0.8702348864073931,
+      "grad_norm": 0.2141641080379486,
+      "learning_rate": 4.376997004524541e-07,
+      "loss": 0.7575,
+      "step": 2260
+    },
+    {
+      "epoch": 0.8740854832499038,
+      "grad_norm": 0.182010218501091,
+      "learning_rate": 4.12540435970073e-07,
+      "loss": 0.7771,
+      "step": 2270
+    },
+    {
+      "epoch": 0.8779360800924143,
+      "grad_norm": 0.1744946837425232,
+      "learning_rate": 3.880946978614703e-07,
+      "loss": 0.7785,
+      "step": 2280
+    },
+    {
+      "epoch": 0.881786676934925,
+      "grad_norm": 0.1830628365278244,
+      "learning_rate": 3.643662883789878e-07,
+      "loss": 0.7869,
+      "step": 2290
+    },
+    {
+      "epoch": 0.8856372737774355,
+      "grad_norm": 0.1747540682554245,
+      "learning_rate": 3.41358898202781e-07,
+      "loss": 0.7667,
+      "step": 2300
+    },
+    {
+      "epoch": 0.889487870619946,
+      "grad_norm": 0.1819252222776413,
+      "learning_rate": 3.190761058667741e-07,
+      "loss": 0.7775,
+      "step": 2310
+    },
+    {
+      "epoch": 0.8933384674624567,
+      "grad_norm": 0.1740165501832962,
+      "learning_rate": 2.9752137720206463e-07,
+      "loss": 0.7812,
+      "step": 2320
+    },
+    {
+      "epoch": 0.8971890643049673,
+      "grad_norm": 0.19191665947437286,
+      "learning_rate": 2.7669806479785067e-07,
+      "loss": 0.7898,
+      "step": 2330
+    },
+    {
+      "epoch": 0.9010396611474779,
+      "grad_norm": 0.17981885373592377,
+      "learning_rate": 2.566094074799763e-07,
+      "loss": 0.7832,
+      "step": 2340
+    },
+    {
+      "epoch": 0.9048902579899885,
+      "grad_norm": 0.18672721087932587,
+      "learning_rate": 2.3725852980716557e-07,
+      "loss": 0.7608,
+      "step": 2350
+    },
+    {
+      "epoch": 0.908740854832499,
+      "grad_norm": 0.17902259528636932,
+      "learning_rate": 2.1864844158503983e-07,
+      "loss": 0.7761,
+      "step": 2360
+    },
+    {
+      "epoch": 0.9125914516750097,
+      "grad_norm": 0.20852720737457275,
+      "learning_rate": 2.007820373979702e-07,
+      "loss": 0.7696,
+      "step": 2370
+    },
+    {
+      "epoch": 0.9164420485175202,
+      "grad_norm": 0.17944659292697906,
+      "learning_rate": 1.8366209615886532e-07,
+      "loss": 0.779,
+      "step": 2380
+    },
+    {
+      "epoch": 0.9202926453600309,
+      "grad_norm": 0.18704205751419067,
+      "learning_rate": 1.6729128067693613e-07,
+      "loss": 0.7904,
+      "step": 2390
+    },
+    {
+      "epoch": 0.9241432422025414,
+      "grad_norm": 0.20024287700653076,
+      "learning_rate": 1.5167213724353426e-07,
+      "loss": 0.7851,
+      "step": 2400
+    },
+    {
+      "epoch": 0.927993839045052,
+      "grad_norm": 0.18266457319259644,
+      "learning_rate": 1.3680709523610058e-07,
+      "loss": 0.7624,
+      "step": 2410
+    },
+    {
+      "epoch": 0.9318444358875626,
+      "grad_norm": 0.1849660575389862,
+      "learning_rate": 1.2269846674030418e-07,
+      "loss": 0.7947,
+      "step": 2420
+    },
+    {
+      "epoch": 0.9356950327300732,
+      "grad_norm": 0.18617238104343414,
+      "learning_rate": 1.0934844619042595e-07,
+      "loss": 0.7862,
+      "step": 2430
+    },
+    {
+      "epoch": 0.9395456295725837,
+      "grad_norm": 0.18520484864711761,
+      "learning_rate": 9.675911002803961e-08,
+      "loss": 0.7782,
+      "step": 2440
+    },
+    {
+      "epoch": 0.9433962264150944,
+      "grad_norm": 0.19607660174369812,
+      "learning_rate": 8.493241637904381e-08,
+      "loss": 0.7663,
+      "step": 2450
+    },
+    {
+      "epoch": 0.9472468232576049,
+      "grad_norm": 0.1911480724811554,
+      "learning_rate": 7.387020474909689e-08,
+      "loss": 0.7929,
+      "step": 2460
+    },
+    {
+      "epoch": 0.9510974201001156,
+      "grad_norm": 0.1945187896490097,
+      "learning_rate": 6.357419573750856e-08,
+      "loss": 0.7887,
+      "step": 2470
+    },
+    {
+      "epoch": 0.9549480169426261,
+      "grad_norm": 0.18382154405117035,
+      "learning_rate": 5.4045990769613365e-08,
+      "loss": 0.7745,
+      "step": 2480
+    },
+    {
+      "epoch": 0.9587986137851366,
+      "grad_norm": 0.18443989753723145,
+      "learning_rate": 4.5287071847694434e-08,
+      "loss": 0.7756,
+      "step": 2490
+    },
+    {
+      "epoch": 0.9626492106276473,
+      "grad_norm": 0.1814785599708557,
+      "learning_rate": 3.7298801320467817e-08,
+      "loss": 0.7757,
+      "step": 2500
+    },
+    {
+      "epoch": 0.9664998074701578,
+      "grad_norm": 0.18538948893547058,
+      "learning_rate": 3.0082421671192576e-08,
+      "loss": 0.775,
+      "step": 2510
+    },
+    {
+      "epoch": 0.9703504043126685,
+      "grad_norm": 0.19164584577083588,
+      "learning_rate": 2.3639055324410355e-08,
+      "loss": 0.7886,
+      "step": 2520
+    },
+    {
+      "epoch": 0.974201001155179,
+      "grad_norm": 0.19045446813106537,
+      "learning_rate": 1.7969704471369477e-08,
+      "loss": 0.7705,
+      "step": 2530
+    },
+    {
+      "epoch": 0.9780515979976896,
+      "grad_norm": 0.17372433841228485,
+      "learning_rate": 1.3075250914142413e-08,
+      "loss": 0.7698,
+      "step": 2540
+    },
+    {
+      "epoch": 0.9819021948402002,
+      "grad_norm": 0.1751081943511963,
+      "learning_rate": 8.956455928476049e-09,
+      "loss": 0.7868,
+      "step": 2550
+    },
+    {
+      "epoch": 0.9857527916827108,
+      "grad_norm": 0.17903444170951843,
+      "learning_rate": 5.6139601453786275e-09,
+      "loss": 0.772,
+      "step": 2560
+    },
+    {
+      "epoch": 0.9896033885252215,
+      "grad_norm": 0.18579405546188354,
+      "learning_rate": 3.0482834514805602e-09,
+      "loss": 0.767,
+      "step": 2570
+    },
+    {
+      "epoch": 0.993453985367732,
+      "grad_norm": 0.1780356764793396,
+      "learning_rate": 1.2598249081713366e-09,
+      "loss": 0.7587,
+      "step": 2580
+    },
+    {
+      "epoch": 0.9973045822102425,
+      "grad_norm": 0.19472339749336243,
+      "learning_rate": 2.488626895302826e-10,
+      "loss": 0.7893,
+      "step": 2590
+    },
+    {
+      "epoch": 1.0,
+      "step": 2597,
+      "total_flos": 5.970109330052088e+18,
+      "train_loss": 0.7920872881451982,
+      "train_runtime": 14659.2382,
+      "train_samples_per_second": 45.357,
+      "train_steps_per_second": 0.177
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2597,
+  "num_input_tokens_seen": 584317136,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.970109330052088e+18,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

continuous/smollm2/dino_vitb16/finetune/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:686a310feac41b7e4ab041240745d2e26ac0dfaa5feaa07d713ebcc1dd520981
+size 6264

continuous/smollm2/dino_vitb16/pretrain/config.json

@@ -0,0 +1,70 @@
+{
+  "add_faster_video": false,
+  "add_time_instruction": false,
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "faster_token_stride": 10,
+  "force_sample": false,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_aspect_ratio": "square",
+  "image_crop_resolution": null,
+  "image_grid_pinpoints": null,
+  "image_split_resolution": null,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mm_hidden_size": 768,
+  "mm_newline_position": "grid",
+  "mm_patch_merge_type": "flat",
+  "mm_projector_lr": null,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_resampler_type": null,
+  "mm_spatial_pool_mode": "bilinear",
+  "mm_spatial_pool_stride": null,
+  "mm_tunable_parts": "mm_mlp_adapter",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -1,
+  "mm_vision_tower": "hf:/cache/ckpt/download/tokenizer/continuous/dino/dino-vitb16",
+  "mm_vision_tower_lr": null,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "pos_skipping_range": 4096,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 130000,
+  "tie_word_embeddings": true,
+  "tokenizer_model_max_length": 8192,
+  "tokenizer_padding_side": "right",
+  "transformers.js_config": {
+    "dtype": "q4",
+    "kv_cache_dtype": {
+      "fp16": "float16",
+      "q4f16": "float16"
+    },
+    "use_external_data_format": {
+      "model.onnx": true,
+      "model_fp16.onnx": true
+    }
+  },
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "use_mm_proj": true,
+  "use_pos_skipping": false,
+  "vision_tower_pretrained": null,
+  "vocab_size": 49152
+}

continuous/smollm2/dino_vitb16/pretrain/mm_projector.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca2c79bfd1515cd91909de6d9668a55c8c26d80f7271263010478a0e60784c88
+size 23087160

continuous/smollm2/dino_vitb16/pretrain/trainer_state.json

@@ -0,0 +1,421 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 545,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01834862385321101,
+      "grad_norm": 2.25142502784729,
+      "learning_rate": 0.00032142857142857147,
+      "loss": 3.6351,
+      "step": 10
+    },
+    {
+      "epoch": 0.03669724770642202,
+      "grad_norm": 0.16944359242916107,
+      "learning_rate": 0.0006785714285714287,
+      "loss": 3.016,
+      "step": 20
+    },
+    {
+      "epoch": 0.05504587155963303,
+      "grad_norm": 0.11802974343299866,
+      "learning_rate": 0.0009999907688176173,
+      "loss": 2.8525,
+      "step": 30
+    },
+    {
+      "epoch": 0.07339449541284404,
+      "grad_norm": 0.1000688374042511,
+      "learning_rate": 0.0009988834393115766,
+      "loss": 2.7736,
+      "step": 40
+    },
+    {
+      "epoch": 0.09174311926605505,
+      "grad_norm": 0.13677899539470673,
+      "learning_rate": 0.0009959345572940772,
+      "loss": 2.7383,
+      "step": 50
+    },
+    {
+      "epoch": 0.11009174311926606,
+      "grad_norm": 0.11989624053239822,
+      "learning_rate": 0.0009911550081155983,
+      "loss": 2.701,
+      "step": 60
+    },
+    {
+      "epoch": 0.12844036697247707,
+      "grad_norm": 0.11149061471223831,
+      "learning_rate": 0.0009845624347566433,
+      "loss": 2.6773,
+      "step": 70
+    },
+    {
+      "epoch": 0.14678899082568808,
+      "grad_norm": 0.1347535103559494,
+      "learning_rate": 0.0009761811727013547,
+      "loss": 2.658,
+      "step": 80
+    },
+    {
+      "epoch": 0.1651376146788991,
+      "grad_norm": 0.13391529023647308,
+      "learning_rate": 0.0009660421601067666,
+      "loss": 2.6436,
+      "step": 90
+    },
+    {
+      "epoch": 0.1834862385321101,
+      "grad_norm": 0.2515474259853363,
+      "learning_rate": 0.0009541828235992926,
+      "loss": 2.6361,
+      "step": 100
+    },
+    {
+      "epoch": 0.2018348623853211,
+      "grad_norm": 0.11320880055427551,
+      "learning_rate": 0.0009406469401200151,
+      "loss": 2.6242,
+      "step": 110
+    },
+    {
+      "epoch": 0.22018348623853212,
+      "grad_norm": 0.09884700924158096,
+      "learning_rate": 0.0009254844753287492,
+      "loss": 2.6152,
+      "step": 120
+    },
+    {
+      "epoch": 0.23853211009174313,
+      "grad_norm": 0.09119711071252823,
+      "learning_rate": 0.0009087513991633924,
+      "loss": 2.5964,
+      "step": 130
+    },
+    {
+      "epoch": 0.25688073394495414,
+      "grad_norm": 0.10720255970954895,
+      "learning_rate": 0.0008905094792353917,
+      "loss": 2.5869,
+      "step": 140
+    },
+    {
+      "epoch": 0.27522935779816515,
+      "grad_norm": 0.16686685383319855,
+      "learning_rate": 0.0008708260528239789,
+      "loss": 2.5922,
+      "step": 150
+    },
+    {
+      "epoch": 0.29357798165137616,
+      "grad_norm": 0.10048136115074158,
+      "learning_rate": 0.0008497737783108237,
+      "loss": 2.5858,
+      "step": 160
+    },
+    {
+      "epoch": 0.3119266055045872,
+      "grad_norm": 0.09044550359249115,
+      "learning_rate": 0.0008274303669726426,
+      "loss": 2.5803,
+      "step": 170
+    },
+    {
+      "epoch": 0.3302752293577982,
+      "grad_norm": 0.0988573282957077,
+      "learning_rate": 0.0008038782961218137,
+      "loss": 2.5727,
+      "step": 180
+    },
+    {
+      "epoch": 0.3486238532110092,
+      "grad_norm": 0.09711358696222305,
+      "learning_rate": 0.000779204504653894,
+      "loss": 2.5706,
+      "step": 190
+    },
+    {
+      "epoch": 0.3669724770642202,
+      "grad_norm": 0.0933903306722641,
+      "learning_rate": 0.0007535000721258791,
+      "loss": 2.5513,
+      "step": 200
+    },
+    {
+      "epoch": 0.3853211009174312,
+      "grad_norm": 0.13185639679431915,
+      "learning_rate": 0.00072685988254984,
+      "loss": 2.5526,
+      "step": 210
+    },
+    {
+      "epoch": 0.4036697247706422,
+      "grad_norm": 0.09200277179479599,
+      "learning_rate": 0.0006993822741429906,
+      "loss": 2.5597,
+      "step": 220
+    },
+    {
+      "epoch": 0.42201834862385323,
+      "grad_norm": 0.09122661501169205,
+      "learning_rate": 0.0006711686763270818,
+      "loss": 2.5577,
+      "step": 230
+    },
+    {
+      "epoch": 0.44036697247706424,
+      "grad_norm": 0.11650240421295166,
+      "learning_rate": 0.0006423232353170831,
+      "loss": 2.5485,
+      "step": 240
+    },
+    {
+      "epoch": 0.45871559633027525,
+      "grad_norm": 0.11517910659313202,
+      "learning_rate": 0.0006129524296812335,
+      "loss": 2.5433,
+      "step": 250
+    },
+    {
+      "epoch": 0.47706422018348627,
+      "grad_norm": 0.09370420128107071,
+      "learning_rate": 0.0005831646772915651,
+      "loss": 2.5492,
+      "step": 260
+    },
+    {
+      "epoch": 0.4954128440366973,
+      "grad_norm": 0.1422043889760971,
+      "learning_rate": 0.0005530699351157809,
+      "loss": 2.5308,
+      "step": 270
+    },
+    {
+      "epoch": 0.5137614678899083,
+      "grad_norm": 0.10177354514598846,
+      "learning_rate": 0.0005227792933277943,
+      "loss": 2.5368,
+      "step": 280
+    },
+    {
+      "epoch": 0.5321100917431193,
+      "grad_norm": 0.09793924540281296,
+      "learning_rate": 0.0004924045652352092,
+      "loss": 2.5217,
+      "step": 290
+    },
+    {
+      "epoch": 0.5504587155963303,
+      "grad_norm": 0.0909055769443512,
+      "learning_rate": 0.0004620578745374633,
+      "loss": 2.5222,
+      "step": 300
+    },
+    {
+      "epoch": 0.5688073394495413,
+      "grad_norm": 0.11370716243982315,
+      "learning_rate": 0.0004318512414382059,
+      "loss": 2.5306,
+      "step": 310
+    },
+    {
+      "epoch": 0.5871559633027523,
+      "grad_norm": 0.0927586704492569,
+      "learning_rate": 0.0004018961691397148,
+      "loss": 2.5304,
+      "step": 320
+    },
+    {
+      "epoch": 0.6055045871559633,
+      "grad_norm": 0.1070285215973854,
+      "learning_rate": 0.0003723032322457458,
+      "loss": 2.5138,
+      "step": 330
+    },
+    {
+      "epoch": 0.6238532110091743,
+      "grad_norm": 0.14404618740081787,
+      "learning_rate": 0.0003431816685921625,
+      "loss": 2.5175,
+      "step": 340
+    },
+    {
+      "epoch": 0.6422018348623854,
+      "grad_norm": 0.09165588021278381,
+      "learning_rate": 0.0003146389760120416,
+      "loss": 2.5213,
+      "step": 350
+    },
+    {
+      "epoch": 0.6605504587155964,
+      "grad_norm": 0.09375973790884018,
+      "learning_rate": 0.00028678051552373485,
+      "loss": 2.525,
+      "step": 360
+    },
+    {
+      "epoch": 0.6788990825688074,
+      "grad_norm": 0.08602260798215866,
+      "learning_rate": 0.0002597091224066581,
+      "loss": 2.5149,
+      "step": 370
+    },
+    {
+      "epoch": 0.6972477064220184,
+      "grad_norm": 0.07907214015722275,
+      "learning_rate": 0.0002335247266004629,
+      "loss": 2.5272,
+      "step": 380
+    },
+    {
+      "epoch": 0.7155963302752294,
+      "grad_norm": 0.0913601964712143,
+      "learning_rate": 0.000208323983828828,
+      "loss": 2.5115,
+      "step": 390
+    },
+    {
+      "epoch": 0.7339449541284404,
+      "grad_norm": 0.07941664010286331,
+      "learning_rate": 0.0001841999188095224,
+      "loss": 2.5104,
+      "step": 400
+    },
+    {
+      "epoch": 0.7522935779816514,
+      "grad_norm": 0.08359213173389435,
+      "learning_rate": 0.00016124158186777676,
+      "loss": 2.5074,
+      "step": 410
+    },
+    {
+      "epoch": 0.7706422018348624,
+      "grad_norm": 0.08202653378248215,
+      "learning_rate": 0.0001395337202205211,
+      "loss": 2.5033,
+      "step": 420
+    },
+    {
+      "epoch": 0.7889908256880734,
+      "grad_norm": 0.07895301282405853,
+      "learning_rate": 0.00011915646514489292,
+      "loss": 2.5073,
+      "step": 430
+    },
+    {
+      "epoch": 0.8073394495412844,
+      "grad_norm": 0.08526412397623062,
+      "learning_rate": 0.00010018503618578818,
+      "loss": 2.5136,
+      "step": 440
+    },
+    {
+      "epoch": 0.8256880733944955,
+      "grad_norm": 0.07965277880430222,
+      "learning_rate": 8.268946349432583e-05,
+      "loss": 2.5124,
+      "step": 450
+    },
+    {
+      "epoch": 0.8440366972477065,
+      "grad_norm": 0.08328872174024582,
+      "learning_rate": 6.673432932217117e-05,
+      "loss": 2.5017,
+      "step": 460
+    },
+    {
+      "epoch": 0.8623853211009175,
+      "grad_norm": 0.08611209690570831,
+      "learning_rate": 5.2378529625954684e-05,
+      "loss": 2.493,
+      "step": 470
+    },
+    {
+      "epoch": 0.8807339449541285,
+      "grad_norm": 0.08181488513946533,
+      "learning_rate": 3.9675056661785556e-05,
+      "loss": 2.499,
+      "step": 480
+    },
+    {
+      "epoch": 0.8990825688073395,
+      "grad_norm": 0.07460423558950424,
+      "learning_rate": 2.8670803372379538e-05,
+      "loss": 2.5048,
+      "step": 490
+    },
+    {
+      "epoch": 0.9174311926605505,
+      "grad_norm": 0.07293371111154556,
+      "learning_rate": 1.9406390288876586e-05,
+      "loss": 2.508,
+      "step": 500
+    },
+    {
+      "epoch": 0.9357798165137615,
+      "grad_norm": 0.07729075103998184,
+      "learning_rate": 1.191601558631461e-05,
+      "loss": 2.4981,
+      "step": 510
+    },
+    {
+      "epoch": 0.9541284403669725,
+      "grad_norm": 0.07808642834424973,
+      "learning_rate": 6.2273288462559305e-06,
+      "loss": 2.4944,
+      "step": 520
+    },
+    {
+      "epoch": 0.9724770642201835,
+      "grad_norm": 0.077662892639637,
+      "learning_rate": 2.361328992552314e-06,
+      "loss": 2.4966,
+      "step": 530
+    },
+    {
+      "epoch": 0.9908256880733946,
+      "grad_norm": 0.07702932506799698,
+      "learning_rate": 3.322867770044202e-07,
+      "loss": 2.4994,
+      "step": 540
+    },
+    {
+      "epoch": 1.0,
+      "step": 545,
+      "total_flos": 3.541109592303862e+17,
+      "train_loss": 2.589691013371179,
+      "train_runtime": 1378.2568,
+      "train_samples_per_second": 404.952,
+      "train_steps_per_second": 0.395
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 545,
+  "num_input_tokens_seen": 34658176,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.541109592303862e+17,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

continuous/smollm2/dino_vitb8/finetune/chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

continuous/smollm2/dino_vitb8/finetune/config.json

@@ -0,0 +1,70 @@
+{
+  "add_faster_video": false,
+  "add_time_instruction": false,
+  "architectures": [
+    "LlavaLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "faster_token_stride": 10,
+  "force_sample": false,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_aspect_ratio": "square",
+  "image_crop_resolution": null,
+  "image_grid_pinpoints": null,
+  "image_split_resolution": null,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mm_hidden_size": 768,
+  "mm_newline_position": "grid",
+  "mm_patch_merge_type": "flat",
+  "mm_projector_lr": 0.0001,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_resampler_type": null,
+  "mm_spatial_pool_mode": "bilinear",
+  "mm_spatial_pool_stride": null,
+  "mm_tunable_parts": "mm_mlp_adapter,mm_language_model",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -1,
+  "mm_vision_tower": "hf:/cache/ckpt/download/tokenizer/continuous/dino/dino-vitb8",
+  "mm_vision_tower_lr": null,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "pos_skipping_range": 4096,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 130000,
+  "tie_word_embeddings": true,
+  "tokenizer_model_max_length": 8192,
+  "tokenizer_padding_side": "right",
+  "transformers.js_config": {
+    "dtype": "q4",
+    "kv_cache_dtype": {
+      "fp16": "float16",
+      "q4f16": "float16"
+    },
+    "use_external_data_format": {
+      "model.onnx": true,
+      "model_fp16.onnx": true
+    }
+  },
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "use_mm_proj": true,
+  "use_pos_skipping": false,
+  "vision_tower_pretrained": null,
+  "vocab_size": 49152
+}

continuous/smollm2/dino_vitb8/finetune/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    2
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.57.6"
+}

continuous/smollm2/dino_vitb8/finetune/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6805c53bb38872faa03c307bb46fde9324aa89affd24a8875d0c1d8f8b5fbc6f
+size 3820016576

continuous/smollm2/dino_vitb8/finetune/special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

continuous/smollm2/dino_vitb8/finetune/tokenizer_config.json

@@ -0,0 +1,155 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

continuous/smollm2/dino_vitb8/finetune/trainer_state.json

@@ -0,0 +1,1856 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 2597,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.003850596842510589,
+      "grad_norm": 3.002810001373291,
+      "learning_rate": 1.153846153846154e-06,
+      "loss": 1.2067,
+      "step": 10
+    },
+    {
+      "epoch": 0.007701193685021178,
+      "grad_norm": 2.071232795715332,
+      "learning_rate": 2.435897435897436e-06,
+      "loss": 1.1056,
+      "step": 20
+    },
+    {
+      "epoch": 0.011551790527531768,
+      "grad_norm": 1.9870126247406006,
+      "learning_rate": 3.7179487179487184e-06,
+      "loss": 1.0265,
+      "step": 30
+    },
+    {
+      "epoch": 0.015402387370042356,
+      "grad_norm": 1.3564453125,
+      "learning_rate": 5e-06,
+      "loss": 0.9858,
+      "step": 40
+    },
+    {
+      "epoch": 0.019252984212552945,
+      "grad_norm": 1.2921870946884155,
+      "learning_rate": 6.282051282051282e-06,
+      "loss": 0.9725,
+      "step": 50
+    },
+    {
+      "epoch": 0.023103581055063535,
+      "grad_norm": 0.33536961674690247,
+      "learning_rate": 7.564102564102564e-06,
+      "loss": 0.9055,
+      "step": 60
+    },
+    {
+      "epoch": 0.026954177897574125,
+      "grad_norm": 0.2883870005607605,
+      "learning_rate": 8.846153846153847e-06,
+      "loss": 0.8594,
+      "step": 70
+    },
+    {
+      "epoch": 0.030804774740084712,
+      "grad_norm": 0.2929156422615051,
+      "learning_rate": 9.999996111488723e-06,
+      "loss": 0.8749,
+      "step": 80
+    },
+    {
+      "epoch": 0.0346553715825953,
+      "grad_norm": 0.3067626655101776,
+      "learning_rate": 9.999529497453782e-06,
+      "loss": 0.862,
+      "step": 90
+    },
+    {
+      "epoch": 0.03850596842510589,
+      "grad_norm": 0.27555370330810547,
+      "learning_rate": 9.998285264324193e-06,
+      "loss": 0.8326,
+      "step": 100
+    },
+    {
+      "epoch": 0.04235656526761648,
+      "grad_norm": 0.30644863843917847,
+      "learning_rate": 9.996263605626052e-06,
+      "loss": 0.8231,
+      "step": 110
+    },
+    {
+      "epoch": 0.04620716211012707,
+      "grad_norm": 0.25902464985847473,
+      "learning_rate": 9.99346483580503e-06,
+      "loss": 0.843,
+      "step": 120
+    },
+    {
+      "epoch": 0.05005775895263766,
+      "grad_norm": 0.27868276834487915,
+      "learning_rate": 9.989889390177467e-06,
+      "loss": 0.8206,
+      "step": 130
+    },
+    {
+      "epoch": 0.05390835579514825,
+      "grad_norm": 0.2553630471229553,
+      "learning_rate": 9.985537824862647e-06,
+      "loss": 0.8332,
+      "step": 140
+    },
+    {
+      "epoch": 0.05775895263765884,
+      "grad_norm": 0.26548993587493896,
+      "learning_rate": 9.980410816696317e-06,
+      "loss": 0.811,
+      "step": 150
+    },
+    {
+      "epoch": 0.061609549480169425,
+      "grad_norm": 0.2551515996456146,
+      "learning_rate": 9.97450916312541e-06,
+      "loss": 0.8165,
+      "step": 160
+    },
+    {
+      "epoch": 0.06546014632268002,
+      "grad_norm": 0.26463159918785095,
+      "learning_rate": 9.967833782084004e-06,
+      "loss": 0.8168,
+      "step": 170
+    },
+    {
+      "epoch": 0.0693107431651906,
+      "grad_norm": 0.28220662474632263,
+      "learning_rate": 9.960385711850554e-06,
+      "loss": 0.8493,
+      "step": 180
+    },
+    {
+      "epoch": 0.07316134000770119,
+      "grad_norm": 0.31214237213134766,
+      "learning_rate": 9.952166110886394e-06,
+      "loss": 0.8366,
+      "step": 190
+    },
+    {
+      "epoch": 0.07701193685021178,
+      "grad_norm": 0.26791244745254517,
+      "learning_rate": 9.943176257655567e-06,
+      "loss": 0.8361,
+      "step": 200
+    },
+    {
+      "epoch": 0.08086253369272237,
+      "grad_norm": 0.2843988835811615,
+      "learning_rate": 9.933417550425952e-06,
+      "loss": 0.821,
+      "step": 210
+    },
+    {
+      "epoch": 0.08471313053523297,
+      "grad_norm": 0.25606539845466614,
+      "learning_rate": 9.9228915070518e-06,
+      "loss": 0.8143,
+      "step": 220
+    },
+    {
+      "epoch": 0.08856372737774355,
+      "grad_norm": 0.24536281824111938,
+      "learning_rate": 9.911599764737632e-06,
+      "loss": 0.8087,
+      "step": 230
+    },
+    {
+      "epoch": 0.09241432422025414,
+      "grad_norm": 0.30814605951309204,
+      "learning_rate": 9.899544079783606e-06,
+      "loss": 0.802,
+      "step": 240
+    },
+    {
+      "epoch": 0.09626492106276473,
+      "grad_norm": 0.2989020347595215,
+      "learning_rate": 9.886726327312338e-06,
+      "loss": 0.82,
+      "step": 250
+    },
+    {
+      "epoch": 0.10011551790527531,
+      "grad_norm": 0.25837647914886475,
+      "learning_rate": 9.87314850097724e-06,
+      "loss": 0.8195,
+      "step": 260
+    },
+    {
+      "epoch": 0.1039661147477859,
+      "grad_norm": 0.23201341927051544,
+      "learning_rate": 9.858812712652452e-06,
+      "loss": 0.8004,
+      "step": 270
+    },
+    {
+      "epoch": 0.1078167115902965,
+      "grad_norm": 0.2542673945426941,
+      "learning_rate": 9.843721192104341e-06,
+      "loss": 0.7994,
+      "step": 280
+    },
+    {
+      "epoch": 0.11166730843280709,
+      "grad_norm": 0.2838711440563202,
+      "learning_rate": 9.827876286644698e-06,
+      "loss": 0.8308,
+      "step": 290
+    },
+    {
+      "epoch": 0.11551790527531768,
+      "grad_norm": 0.2264915406703949,
+      "learning_rate": 9.811280460765643e-06,
+      "loss": 0.7884,
+      "step": 300
+    },
+    {
+      "epoch": 0.11936850211782826,
+      "grad_norm": 0.26007726788520813,
+      "learning_rate": 9.793936295756292e-06,
+      "loss": 0.7939,
+      "step": 310
+    },
+    {
+      "epoch": 0.12321909896033885,
+      "grad_norm": 0.24438567459583282,
+      "learning_rate": 9.775846489301279e-06,
+      "loss": 0.8024,
+      "step": 320
+    },
+    {
+      "epoch": 0.12706969580284944,
+      "grad_norm": 0.23977306485176086,
+      "learning_rate": 9.757013855061157e-06,
+      "loss": 0.8056,
+      "step": 330
+    },
+    {
+      "epoch": 0.13092029264536004,
+      "grad_norm": 0.22502347826957703,
+      "learning_rate": 9.737441322234758e-06,
+      "loss": 0.8062,
+      "step": 340
+    },
+    {
+      "epoch": 0.1347708894878706,
+      "grad_norm": 0.25818657875061035,
+      "learning_rate": 9.717131935103605e-06,
+      "loss": 0.8137,
+      "step": 350
+    },
+    {
+      "epoch": 0.1386214863303812,
+      "grad_norm": 0.26462459564208984,
+      "learning_rate": 9.696088852558398e-06,
+      "loss": 0.7937,
+      "step": 360
+    },
+    {
+      "epoch": 0.1424720831728918,
+      "grad_norm": 0.2439662367105484,
+      "learning_rate": 9.674315347607686e-06,
+      "loss": 0.8222,
+      "step": 370
+    },
+    {
+      "epoch": 0.14632268001540238,
+      "grad_norm": 0.32908451557159424,
+      "learning_rate": 9.651814806868796e-06,
+      "loss": 0.787,
+      "step": 380
+    },
+    {
+      "epoch": 0.15017327685791299,
+      "grad_norm": 0.2903500497341156,
+      "learning_rate": 9.62859073004107e-06,
+      "loss": 0.7912,
+      "step": 390
+    },
+    {
+      "epoch": 0.15402387370042356,
+      "grad_norm": 0.21470235288143158,
+      "learning_rate": 9.604646729361547e-06,
+      "loss": 0.7998,
+      "step": 400
+    },
+    {
+      "epoch": 0.15787447054293416,
+      "grad_norm": 0.25997793674468994,
+      "learning_rate": 9.579986529043099e-06,
+      "loss": 0.7962,
+      "step": 410
+    },
+    {
+      "epoch": 0.16172506738544473,
+      "grad_norm": 0.21830108761787415,
+      "learning_rate": 9.554613964695189e-06,
+      "loss": 0.81,
+      "step": 420
+    },
+    {
+      "epoch": 0.16557566422795533,
+      "grad_norm": 0.21277837455272675,
+      "learning_rate": 9.528532982727281e-06,
+      "loss": 0.7883,
+      "step": 430
+    },
+    {
+      "epoch": 0.16942626107046593,
+      "grad_norm": 0.2261422574520111,
+      "learning_rate": 9.501747639735018e-06,
+      "loss": 0.7988,
+      "step": 440
+    },
+    {
+      "epoch": 0.1732768579129765,
+      "grad_norm": 0.26509857177734375,
+      "learning_rate": 9.474262101869272e-06,
+      "loss": 0.7823,
+      "step": 450
+    },
+    {
+      "epoch": 0.1771274547554871,
+      "grad_norm": 0.25518524646759033,
+      "learning_rate": 9.446080644188141e-06,
+      "loss": 0.7914,
+      "step": 460
+    },
+    {
+      "epoch": 0.18097805159799768,
+      "grad_norm": 0.28256845474243164,
+      "learning_rate": 9.417207649992017e-06,
+      "loss": 0.795,
+      "step": 470
+    },
+    {
+      "epoch": 0.18482864844050828,
+      "grad_norm": 0.2647364139556885,
+      "learning_rate": 9.387647610141816e-06,
+      "loss": 0.7951,
+      "step": 480
+    },
+    {
+      "epoch": 0.18867924528301888,
+      "grad_norm": 0.2398519515991211,
+      "learning_rate": 9.35740512236047e-06,
+      "loss": 0.7874,
+      "step": 490
+    },
+    {
+      "epoch": 0.19252984212552945,
+      "grad_norm": 0.22883296012878418,
+      "learning_rate": 9.326484890517809e-06,
+      "loss": 0.7894,
+      "step": 500
+    },
+    {
+      "epoch": 0.19638043896804006,
+      "grad_norm": 0.2151322215795517,
+      "learning_rate": 9.294891723898926e-06,
+      "loss": 0.7808,
+      "step": 510
+    },
+    {
+      "epoch": 0.20023103581055063,
+      "grad_norm": 0.24800144135951996,
+      "learning_rate": 9.262630536456153e-06,
+      "loss": 0.8051,
+      "step": 520
+    },
+    {
+      "epoch": 0.20408163265306123,
+      "grad_norm": 0.24546891450881958,
+      "learning_rate": 9.229706346044749e-06,
+      "loss": 0.7881,
+      "step": 530
+    },
+    {
+      "epoch": 0.2079322294955718,
+      "grad_norm": 0.21965399384498596,
+      "learning_rate": 9.196124273642422e-06,
+      "loss": 0.797,
+      "step": 540
+    },
+    {
+      "epoch": 0.2117828263380824,
+      "grad_norm": 0.2092410922050476,
+      "learning_rate": 9.161889542552842e-06,
+      "loss": 0.8016,
+      "step": 550
+    },
+    {
+      "epoch": 0.215633423180593,
+      "grad_norm": 0.24090446531772614,
+      "learning_rate": 9.127007477593193e-06,
+      "loss": 0.8061,
+      "step": 560
+    },
+    {
+      "epoch": 0.21948402002310358,
+      "grad_norm": 0.24421872198581696,
+      "learning_rate": 9.091483504265974e-06,
+      "loss": 0.8236,
+      "step": 570
+    },
+    {
+      "epoch": 0.22333461686561418,
+      "grad_norm": 0.25984838604927063,
+      "learning_rate": 9.055323147915116e-06,
+      "loss": 0.7863,
+      "step": 580
+    },
+    {
+      "epoch": 0.22718521370812475,
+      "grad_norm": 0.2625979781150818,
+      "learning_rate": 9.018532032866585e-06,
+      "loss": 0.805,
+      "step": 590
+    },
+    {
+      "epoch": 0.23103581055063535,
+      "grad_norm": 0.21989835798740387,
+      "learning_rate": 8.981115881553579e-06,
+      "loss": 0.8022,
+      "step": 600
+    },
+    {
+      "epoch": 0.23488640739314592,
+      "grad_norm": 0.2465246617794037,
+      "learning_rate": 8.943080513626476e-06,
+      "loss": 0.8196,
+      "step": 610
+    },
+    {
+      "epoch": 0.23873700423565652,
+      "grad_norm": 0.20237061381340027,
+      "learning_rate": 8.904431845047641e-06,
+      "loss": 0.7975,
+      "step": 620
+    },
+    {
+      "epoch": 0.24258760107816713,
+      "grad_norm": 0.28781214356422424,
+      "learning_rate": 8.86517588717128e-06,
+      "loss": 0.81,
+      "step": 630
+    },
+    {
+      "epoch": 0.2464381979206777,
+      "grad_norm": 0.22937466204166412,
+      "learning_rate": 8.82531874580844e-06,
+      "loss": 0.8046,
+      "step": 640
+    },
+    {
+      "epoch": 0.25028879476318827,
+      "grad_norm": 0.21160949766635895,
+      "learning_rate": 8.784866620277314e-06,
+      "loss": 0.766,
+      "step": 650
+    },
+    {
+      "epoch": 0.25413939160569887,
+      "grad_norm": 0.24520504474639893,
+      "learning_rate": 8.743825802439018e-06,
+      "loss": 0.7956,
+      "step": 660
+    },
+    {
+      "epoch": 0.2579899884482095,
+      "grad_norm": 0.22692610323429108,
+      "learning_rate": 8.702202675718956e-06,
+      "loss": 0.8008,
+      "step": 670
+    },
+    {
+      "epoch": 0.2618405852907201,
+      "grad_norm": 0.23165082931518555,
+      "learning_rate": 8.660003714113959e-06,
+      "loss": 0.7848,
+      "step": 680
+    },
+    {
+      "epoch": 0.2656911821332307,
+      "grad_norm": 0.23203052580356598,
+      "learning_rate": 8.617235481185323e-06,
+      "loss": 0.7959,
+      "step": 690
+    },
+    {
+      "epoch": 0.2695417789757412,
+      "grad_norm": 0.22868932783603668,
+      "learning_rate": 8.573904629037931e-06,
+      "loss": 0.8062,
+      "step": 700
+    },
+    {
+      "epoch": 0.2733923758182518,
+      "grad_norm": 0.202736034989357,
+      "learning_rate": 8.530017897285585e-06,
+      "loss": 0.786,
+      "step": 710
+    },
+    {
+      "epoch": 0.2772429726607624,
+      "grad_norm": 0.21074575185775757,
+      "learning_rate": 8.485582112002746e-06,
+      "loss": 0.7877,
+      "step": 720
+    },
+    {
+      "epoch": 0.281093569503273,
+      "grad_norm": 0.23523427546024323,
+      "learning_rate": 8.440604184662811e-06,
+      "loss": 0.7983,
+      "step": 730
+    },
+    {
+      "epoch": 0.2849441663457836,
+      "grad_norm": 0.24407894909381866,
+      "learning_rate": 8.395091111063118e-06,
+      "loss": 0.7849,
+      "step": 740
+    },
+    {
+      "epoch": 0.28879476318829417,
+      "grad_norm": 0.20657837390899658,
+      "learning_rate": 8.349049970236822e-06,
+      "loss": 0.7798,
+      "step": 750
+    },
+    {
+      "epoch": 0.29264536003080477,
+      "grad_norm": 0.2479284256696701,
+      "learning_rate": 8.302487923351842e-06,
+      "loss": 0.7779,
+      "step": 760
+    },
+    {
+      "epoch": 0.29649595687331537,
+      "grad_norm": 0.2556270360946655,
+      "learning_rate": 8.255412212597021e-06,
+      "loss": 0.7967,
+      "step": 770
+    },
+    {
+      "epoch": 0.30034655371582597,
+      "grad_norm": 0.2290734350681305,
+      "learning_rate": 8.207830160055688e-06,
+      "loss": 0.7906,
+      "step": 780
+    },
+    {
+      "epoch": 0.3041971505583365,
+      "grad_norm": 0.2148728370666504,
+      "learning_rate": 8.159749166566786e-06,
+      "loss": 0.7764,
+      "step": 790
+    },
+    {
+      "epoch": 0.3080477474008471,
+      "grad_norm": 0.2213246375322342,
+      "learning_rate": 8.11117671057377e-06,
+      "loss": 0.7976,
+      "step": 800
+    },
+    {
+      "epoch": 0.3118983442433577,
+      "grad_norm": 0.21787308156490326,
+      "learning_rate": 8.062120346961404e-06,
+      "loss": 0.7968,
+      "step": 810
+    },
+    {
+      "epoch": 0.3157489410858683,
+      "grad_norm": 0.20951104164123535,
+      "learning_rate": 8.012587705880705e-06,
+      "loss": 0.7946,
+      "step": 820
+    },
+    {
+      "epoch": 0.3195995379283789,
+      "grad_norm": 0.23434963822364807,
+      "learning_rate": 7.962586491562146e-06,
+      "loss": 0.7801,
+      "step": 830
+    },
+    {
+      "epoch": 0.32345013477088946,
+      "grad_norm": 0.23803696036338806,
+      "learning_rate": 7.912124481117357e-06,
+      "loss": 0.7789,
+      "step": 840
+    },
+    {
+      "epoch": 0.32730073161340006,
+      "grad_norm": 0.2046729028224945,
+      "learning_rate": 7.861209523329494e-06,
+      "loss": 0.788,
+      "step": 850
+    },
+    {
+      "epoch": 0.33115132845591067,
+      "grad_norm": 0.21009749174118042,
+      "learning_rate": 7.809849537432432e-06,
+      "loss": 0.7818,
+      "step": 860
+    },
+    {
+      "epoch": 0.33500192529842127,
+      "grad_norm": 0.21883703768253326,
+      "learning_rate": 7.758052511879035e-06,
+      "loss": 0.7843,
+      "step": 870
+    },
+    {
+      "epoch": 0.33885252214093187,
+      "grad_norm": 0.2434491217136383,
+      "learning_rate": 7.705826503098642e-06,
+      "loss": 0.787,
+      "step": 880
+    },
+    {
+      "epoch": 0.3427031189834424,
+      "grad_norm": 0.20622345805168152,
+      "learning_rate": 7.653179634243979e-06,
+      "loss": 0.7801,
+      "step": 890
+    },
+    {
+      "epoch": 0.346553715825953,
+      "grad_norm": 0.21366791427135468,
+      "learning_rate": 7.600120093927696e-06,
+      "loss": 0.7851,
+      "step": 900
+    },
+    {
+      "epoch": 0.3504043126684636,
+      "grad_norm": 0.22429142892360687,
+      "learning_rate": 7.546656134948729e-06,
+      "loss": 0.7744,
+      "step": 910
+    },
+    {
+      "epoch": 0.3542549095109742,
+      "grad_norm": 0.21809744834899902,
+      "learning_rate": 7.492796073008661e-06,
+      "loss": 0.779,
+      "step": 920
+    },
+    {
+      "epoch": 0.3581055063534848,
+      "grad_norm": 0.22947679460048676,
+      "learning_rate": 7.438548285418326e-06,
+      "loss": 0.7614,
+      "step": 930
+    },
+    {
+      "epoch": 0.36195610319599536,
+      "grad_norm": 0.21725355088710785,
+      "learning_rate": 7.3839212097948e-06,
+      "loss": 0.7921,
+      "step": 940
+    },
+    {
+      "epoch": 0.36580670003850596,
+      "grad_norm": 0.21490606665611267,
+      "learning_rate": 7.328923342749039e-06,
+      "loss": 0.8087,
+      "step": 950
+    },
+    {
+      "epoch": 0.36965729688101656,
+      "grad_norm": 0.20951154828071594,
+      "learning_rate": 7.273563238564319e-06,
+      "loss": 0.8032,
+      "step": 960
+    },
+    {
+      "epoch": 0.37350789372352716,
+      "grad_norm": 0.23809491097927094,
+      "learning_rate": 7.217849507865724e-06,
+      "loss": 0.7758,
+      "step": 970
+    },
+    {
+      "epoch": 0.37735849056603776,
+      "grad_norm": 0.23145100474357605,
+      "learning_rate": 7.161790816280858e-06,
+      "loss": 0.7815,
+      "step": 980
+    },
+    {
+      "epoch": 0.3812090874085483,
+      "grad_norm": 0.21912193298339844,
+      "learning_rate": 7.1053958830920125e-06,
+      "loss": 0.7723,
+      "step": 990
+    },
+    {
+      "epoch": 0.3850596842510589,
+      "grad_norm": 0.22063760459423065,
+      "learning_rate": 7.0486734798799715e-06,
+      "loss": 0.7805,
+      "step": 1000
+    },
+    {
+      "epoch": 0.3889102810935695,
+      "grad_norm": 0.19222241640090942,
+      "learning_rate": 6.991632429159708e-06,
+      "loss": 0.7996,
+      "step": 1010
+    },
+    {
+      "epoch": 0.3927608779360801,
+      "grad_norm": 0.22965079545974731,
+      "learning_rate": 6.934281603008132e-06,
+      "loss": 0.8003,
+      "step": 1020
+    },
+    {
+      "epoch": 0.39661147477859066,
+      "grad_norm": 0.18432506918907166,
+      "learning_rate": 6.876629921684143e-06,
+      "loss": 0.797,
+      "step": 1030
+    },
+    {
+      "epoch": 0.40046207162110126,
+      "grad_norm": 0.22677335143089294,
+      "learning_rate": 6.818686352241199e-06,
+      "loss": 0.785,
+      "step": 1040
+    },
+    {
+      "epoch": 0.40431266846361186,
+      "grad_norm": 0.20219732820987701,
+      "learning_rate": 6.760459907132573e-06,
+      "loss": 0.7817,
+      "step": 1050
+    },
+    {
+      "epoch": 0.40816326530612246,
+      "grad_norm": 0.19869816303253174,
+      "learning_rate": 6.701959642809589e-06,
+      "loss": 0.7875,
+      "step": 1060
+    },
+    {
+      "epoch": 0.41201386214863306,
+      "grad_norm": 0.24209462106227875,
+      "learning_rate": 6.643194658312987e-06,
+      "loss": 0.7791,
+      "step": 1070
+    },
+    {
+      "epoch": 0.4158644589911436,
+      "grad_norm": 0.20259660482406616,
+      "learning_rate": 6.584174093857676e-06,
+      "loss": 0.8041,
+      "step": 1080
+    },
+    {
+      "epoch": 0.4197150558336542,
+      "grad_norm": 0.2393861711025238,
+      "learning_rate": 6.5249071294110735e-06,
+      "loss": 0.785,
+      "step": 1090
+    },
+    {
+      "epoch": 0.4235656526761648,
+      "grad_norm": 0.2485576868057251,
+      "learning_rate": 6.465402983265274e-06,
+      "loss": 0.7977,
+      "step": 1100
+    },
+    {
+      "epoch": 0.4274162495186754,
+      "grad_norm": 0.2622028589248657,
+      "learning_rate": 6.405670910603243e-06,
+      "loss": 0.7921,
+      "step": 1110
+    },
+    {
+      "epoch": 0.431266846361186,
+      "grad_norm": 0.2615569233894348,
+      "learning_rate": 6.345720202059291e-06,
+      "loss": 0.7893,
+      "step": 1120
+    },
+    {
+      "epoch": 0.43511744320369655,
+      "grad_norm": 0.2278793752193451,
+      "learning_rate": 6.285560182274008e-06,
+      "loss": 0.7692,
+      "step": 1130
+    },
+    {
+      "epoch": 0.43896804004620715,
+      "grad_norm": 0.22633282840251923,
+      "learning_rate": 6.225200208443936e-06,
+      "loss": 0.7938,
+      "step": 1140
+    },
+    {
+      "epoch": 0.44281863688871775,
+      "grad_norm": 0.2043883502483368,
+      "learning_rate": 6.16464966886616e-06,
+      "loss": 0.7787,
+      "step": 1150
+    },
+    {
+      "epoch": 0.44666923373122835,
+      "grad_norm": 0.24961233139038086,
+      "learning_rate": 6.103917981478065e-06,
+      "loss": 0.8076,
+      "step": 1160
+    },
+    {
+      "epoch": 0.45051983057373896,
+      "grad_norm": 0.22041364014148712,
+      "learning_rate": 6.043014592392489e-06,
+      "loss": 0.8153,
+      "step": 1170
+    },
+    {
+      "epoch": 0.4543704274162495,
+      "grad_norm": 0.21269932389259338,
+      "learning_rate": 5.981948974428485e-06,
+      "loss": 0.7934,
+      "step": 1180
+    },
+    {
+      "epoch": 0.4582210242587601,
+      "grad_norm": 0.20076078176498413,
+      "learning_rate": 5.920730625637934e-06,
+      "loss": 0.7733,
+      "step": 1190
+    },
+    {
+      "epoch": 0.4620716211012707,
+      "grad_norm": 0.22595837712287903,
+      "learning_rate": 5.8593690678282355e-06,
+      "loss": 0.7781,
+      "step": 1200
+    },
+    {
+      "epoch": 0.4659222179437813,
+      "grad_norm": 0.21530114114284515,
+      "learning_rate": 5.797873845081301e-06,
+      "loss": 0.7767,
+      "step": 1210
+    },
+    {
+      "epoch": 0.46977281478629185,
+      "grad_norm": 0.22943170368671417,
+      "learning_rate": 5.736254522269075e-06,
+      "loss": 0.8034,
+      "step": 1220
+    },
+    {
+      "epoch": 0.47362341162880245,
+      "grad_norm": 0.20209944248199463,
+      "learning_rate": 5.6745206835658386e-06,
+      "loss": 0.7973,
+      "step": 1230
+    },
+    {
+      "epoch": 0.47747400847131305,
+      "grad_norm": 0.23177942633628845,
+      "learning_rate": 5.612681930957495e-06,
+      "loss": 0.7759,
+      "step": 1240
+    },
+    {
+      "epoch": 0.48132460531382365,
+      "grad_norm": 0.21554039418697357,
+      "learning_rate": 5.550747882748095e-06,
+      "loss": 0.7708,
+      "step": 1250
+    },
+    {
+      "epoch": 0.48517520215633425,
+      "grad_norm": 0.2355380803346634,
+      "learning_rate": 5.4887281720638195e-06,
+      "loss": 0.7824,
+      "step": 1260
+    },
+    {
+      "epoch": 0.4890257989988448,
+      "grad_norm": 0.2310691922903061,
+      "learning_rate": 5.42663244535466e-06,
+      "loss": 0.7629,
+      "step": 1270
+    },
+    {
+      "epoch": 0.4928763958413554,
+      "grad_norm": 0.22120507061481476,
+      "learning_rate": 5.3644703608940205e-06,
+      "loss": 0.7758,
+      "step": 1280
+    },
+    {
+      "epoch": 0.496726992683866,
+      "grad_norm": 0.20914693176746368,
+      "learning_rate": 5.302251587276491e-06,
+      "loss": 0.7886,
+      "step": 1290
+    },
+    {
+      "epoch": 0.5005775895263765,
+      "grad_norm": 0.2051766812801361,
+      "learning_rate": 5.2399858019140005e-06,
+      "loss": 0.779,
+      "step": 1300
+    },
+    {
+      "epoch": 0.5044281863688872,
+      "grad_norm": 0.22473791241645813,
+      "learning_rate": 5.177682689530615e-06,
+      "loss": 0.7958,
+      "step": 1310
+    },
+    {
+      "epoch": 0.5082787832113977,
+      "grad_norm": 0.231491357088089,
+      "learning_rate": 5.11535194065619e-06,
+      "loss": 0.7698,
+      "step": 1320
+    },
+    {
+      "epoch": 0.5121293800539084,
+      "grad_norm": 0.2136174887418747,
+      "learning_rate": 5.053003250119113e-06,
+      "loss": 0.779,
+      "step": 1330
+    },
+    {
+      "epoch": 0.515979976896419,
+      "grad_norm": 0.21144960820674896,
+      "learning_rate": 4.9906463155383925e-06,
+      "loss": 0.7834,
+      "step": 1340
+    },
+    {
+      "epoch": 0.5198305737389295,
+      "grad_norm": 0.23203642666339874,
+      "learning_rate": 4.9282908358153046e-06,
+      "loss": 0.7787,
+      "step": 1350
+    },
+    {
+      "epoch": 0.5236811705814401,
+      "grad_norm": 0.21471726894378662,
+      "learning_rate": 4.8659465096248345e-06,
+      "loss": 0.7966,
+      "step": 1360
+    },
+    {
+      "epoch": 0.5275317674239507,
+      "grad_norm": 0.23394203186035156,
+      "learning_rate": 4.803623033907169e-06,
+      "loss": 0.7908,
+      "step": 1370
+    },
+    {
+      "epoch": 0.5313823642664613,
+      "grad_norm": 0.2025843858718872,
+      "learning_rate": 4.741330102359441e-06,
+      "loss": 0.8023,
+      "step": 1380
+    },
+    {
+      "epoch": 0.5352329611089719,
+      "grad_norm": 0.23949050903320312,
+      "learning_rate": 4.679077403927996e-06,
+      "loss": 0.7784,
+      "step": 1390
+    },
+    {
+      "epoch": 0.5390835579514824,
+      "grad_norm": 0.22480642795562744,
+      "learning_rate": 4.616874621301373e-06,
+      "loss": 0.7787,
+      "step": 1400
+    },
+    {
+      "epoch": 0.5429341547939931,
+      "grad_norm": 0.2436913400888443,
+      "learning_rate": 4.554731429404293e-06,
+      "loss": 0.7955,
+      "step": 1410
+    },
+    {
+      "epoch": 0.5467847516365036,
+      "grad_norm": 0.2484399527311325,
+      "learning_rate": 4.492657493892822e-06,
+      "loss": 0.7814,
+      "step": 1420
+    },
+    {
+      "epoch": 0.5506353484790143,
+      "grad_norm": 0.24449996650218964,
+      "learning_rate": 4.430662469650999e-06,
+      "loss": 0.7936,
+      "step": 1430
+    },
+    {
+      "epoch": 0.5544859453215248,
+      "grad_norm": 0.22788940370082855,
+      "learning_rate": 4.368755999289118e-06,
+      "loss": 0.7923,
+      "step": 1440
+    },
+    {
+      "epoch": 0.5583365421640354,
+      "grad_norm": 0.22179953753948212,
+      "learning_rate": 4.306947711643949e-06,
+      "loss": 0.7787,
+      "step": 1450
+    },
+    {
+      "epoch": 0.562187139006546,
+      "grad_norm": 0.2428441047668457,
+      "learning_rate": 4.245247220281068e-06,
+      "loss": 0.7827,
+      "step": 1460
+    },
+    {
+      "epoch": 0.5660377358490566,
+      "grad_norm": 0.20940645039081573,
+      "learning_rate": 4.183664121999587e-06,
+      "loss": 0.7807,
+      "step": 1470
+    },
+    {
+      "epoch": 0.5698883326915672,
+      "grad_norm": 0.22132141888141632,
+      "learning_rate": 4.122207995339474e-06,
+      "loss": 0.7718,
+      "step": 1480
+    },
+    {
+      "epoch": 0.5737389295340778,
+      "grad_norm": 0.20610855519771576,
+      "learning_rate": 4.060888399091734e-06,
+      "loss": 0.782,
+      "step": 1490
+    },
+    {
+      "epoch": 0.5775895263765883,
+      "grad_norm": 0.20065337419509888,
+      "learning_rate": 3.999714870811639e-06,
+      "loss": 0.7921,
+      "step": 1500
+    },
+    {
+      "epoch": 0.581440123219099,
+      "grad_norm": 0.21424587070941925,
+      "learning_rate": 3.938696925335275e-06,
+      "loss": 0.7865,
+      "step": 1510
+    },
+    {
+      "epoch": 0.5852907200616095,
+      "grad_norm": 0.2123754769563675,
+      "learning_rate": 3.87784405329962e-06,
+      "loss": 0.7948,
+      "step": 1520
+    },
+    {
+      "epoch": 0.5891413169041202,
+      "grad_norm": 0.19947800040245056,
+      "learning_rate": 3.817165719666382e-06,
+      "loss": 0.7962,
+      "step": 1530
+    },
+    {
+      "epoch": 0.5929919137466307,
+      "grad_norm": 0.2051069140434265,
+      "learning_rate": 3.756671362249841e-06,
+      "loss": 0.7948,
+      "step": 1540
+    },
+    {
+      "epoch": 0.5968425105891413,
+      "grad_norm": 0.1947714239358902,
+      "learning_rate": 3.6963703902488907e-06,
+      "loss": 0.7839,
+      "step": 1550
+    },
+    {
+      "epoch": 0.6006931074316519,
+      "grad_norm": 0.206314355134964,
+      "learning_rate": 3.6362721827835655e-06,
+      "loss": 0.7991,
+      "step": 1560
+    },
+    {
+      "epoch": 0.6045437042741625,
+      "grad_norm": 0.2289874404668808,
+      "learning_rate": 3.576386087436212e-06,
+      "loss": 0.8004,
+      "step": 1570
+    },
+    {
+      "epoch": 0.608394301116673,
+      "grad_norm": 0.21806661784648895,
+      "learning_rate": 3.5167214187975913e-06,
+      "loss": 0.7944,
+      "step": 1580
+    },
+    {
+      "epoch": 0.6122448979591837,
+      "grad_norm": 0.1958981305360794,
+      "learning_rate": 3.457287457018086e-06,
+      "loss": 0.7879,
+      "step": 1590
+    },
+    {
+      "epoch": 0.6160954948016942,
+      "grad_norm": 0.21973346173763275,
+      "learning_rate": 3.3980934463643003e-06,
+      "loss": 0.7842,
+      "step": 1600
+    },
+    {
+      "epoch": 0.6199460916442049,
+      "grad_norm": 0.20109491050243378,
+      "learning_rate": 3.3391485937812053e-06,
+      "loss": 0.762,
+      "step": 1610
+    },
+    {
+      "epoch": 0.6237966884867154,
+      "grad_norm": 0.2208002656698227,
+      "learning_rate": 3.280462067460115e-06,
+      "loss": 0.7639,
+      "step": 1620
+    },
+    {
+      "epoch": 0.627647285329226,
+      "grad_norm": 0.21399201452732086,
+      "learning_rate": 3.222042995412669e-06,
+      "loss": 0.7887,
+      "step": 1630
+    },
+    {
+      "epoch": 0.6314978821717366,
+      "grad_norm": 0.18345606327056885,
+      "learning_rate": 3.163900464051089e-06,
+      "loss": 0.7805,
+      "step": 1640
+    },
+    {
+      "epoch": 0.6353484790142472,
+      "grad_norm": 0.23258651793003082,
+      "learning_rate": 3.1060435167748826e-06,
+      "loss": 0.7934,
+      "step": 1650
+    },
+    {
+      "epoch": 0.6391990758567578,
+      "grad_norm": 0.2124588042497635,
+      "learning_rate": 3.048481152564253e-06,
+      "loss": 0.7695,
+      "step": 1660
+    },
+    {
+      "epoch": 0.6430496726992684,
+      "grad_norm": 0.22426602244377136,
+      "learning_rate": 2.9912223245804017e-06,
+      "loss": 0.788,
+      "step": 1670
+    },
+    {
+      "epoch": 0.6469002695417789,
+      "grad_norm": 0.2104906588792801,
+      "learning_rate": 2.934275938772981e-06,
+      "loss": 0.7821,
+      "step": 1680
+    },
+    {
+      "epoch": 0.6507508663842896,
+      "grad_norm": 0.22631751000881195,
+      "learning_rate": 2.8776508524948676e-06,
+      "loss": 0.7769,
+      "step": 1690
+    },
+    {
+      "epoch": 0.6546014632268001,
+      "grad_norm": 0.21501468122005463,
+      "learning_rate": 2.821355873124503e-06,
+      "loss": 0.781,
+      "step": 1700
+    },
+    {
+      "epoch": 0.6584520600693108,
+      "grad_norm": 0.24375596642494202,
+      "learning_rate": 2.7653997566960144e-06,
+      "loss": 0.7831,
+      "step": 1710
+    },
+    {
+      "epoch": 0.6623026569118213,
+      "grad_norm": 0.22722619771957397,
+      "learning_rate": 2.709791206537309e-06,
+      "loss": 0.7771,
+      "step": 1720
+    },
+    {
+      "epoch": 0.6661532537543319,
+      "grad_norm": 0.21745002269744873,
+      "learning_rate": 2.6545388719163747e-06,
+      "loss": 0.7857,
+      "step": 1730
+    },
+    {
+      "epoch": 0.6700038505968425,
+      "grad_norm": 0.21025465428829193,
+      "learning_rate": 2.599651346695979e-06,
+      "loss": 0.7927,
+      "step": 1740
+    },
+    {
+      "epoch": 0.6738544474393531,
+      "grad_norm": 0.19328536093235016,
+      "learning_rate": 2.545137167997007e-06,
+      "loss": 0.7837,
+      "step": 1750
+    },
+    {
+      "epoch": 0.6777050442818637,
+      "grad_norm": 0.20528970658779144,
+      "learning_rate": 2.4910048148705972e-06,
+      "loss": 0.7796,
+      "step": 1760
+    },
+    {
+      "epoch": 0.6815556411243743,
+      "grad_norm": 0.23482580482959747,
+      "learning_rate": 2.4372627069793327e-06,
+      "loss": 0.813,
+      "step": 1770
+    },
+    {
+      "epoch": 0.6854062379668848,
+      "grad_norm": 0.20085804164409637,
+      "learning_rate": 2.3839192032876486e-06,
+      "loss": 0.7908,
+      "step": 1780
+    },
+    {
+      "epoch": 0.6892568348093955,
+      "grad_norm": 0.21298010647296906,
+      "learning_rate": 2.3309826007617047e-06,
+      "loss": 0.8061,
+      "step": 1790
+    },
+    {
+      "epoch": 0.693107431651906,
+      "grad_norm": 0.21215873956680298,
+      "learning_rate": 2.278461133078881e-06,
+      "loss": 0.7736,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6969580284944167,
+      "grad_norm": 0.21524125337600708,
+      "learning_rate": 2.2263629693471258e-06,
+      "loss": 0.7688,
+      "step": 1810
+    },
+    {
+      "epoch": 0.7008086253369272,
+      "grad_norm": 0.23042833805084229,
+      "learning_rate": 2.1746962128343414e-06,
+      "loss": 0.7927,
+      "step": 1820
+    },
+    {
+      "epoch": 0.7046592221794378,
+      "grad_norm": 0.22345513105392456,
+      "learning_rate": 2.1234688997080183e-06,
+      "loss": 0.774,
+      "step": 1830
+    },
+    {
+      "epoch": 0.7085098190219484,
+      "grad_norm": 0.2033495008945465,
+      "learning_rate": 2.0726889977853053e-06,
+      "loss": 0.7789,
+      "step": 1840
+    },
+    {
+      "epoch": 0.712360415864459,
+      "grad_norm": 0.21768423914909363,
+      "learning_rate": 2.022364405293703e-06,
+      "loss": 0.793,
+      "step": 1850
+    },
+    {
+      "epoch": 0.7162110127069696,
+      "grad_norm": 0.2090577632188797,
+      "learning_rate": 1.9725029496425814e-06,
+      "loss": 0.7919,
+      "step": 1860
+    },
+    {
+      "epoch": 0.7200616095494802,
+      "grad_norm": 0.20679739117622375,
+      "learning_rate": 1.923112386205727e-06,
+      "loss": 0.7814,
+      "step": 1870
+    },
+    {
+      "epoch": 0.7239122063919907,
+      "grad_norm": 0.2111605852842331,
+      "learning_rate": 1.874200397115079e-06,
+      "loss": 0.7936,
+      "step": 1880
+    },
+    {
+      "epoch": 0.7277628032345014,
+      "grad_norm": 0.22066271305084229,
+      "learning_rate": 1.825774590065857e-06,
+      "loss": 0.7795,
+      "step": 1890
+    },
+    {
+      "epoch": 0.7316134000770119,
+      "grad_norm": 0.23580186069011688,
+      "learning_rate": 1.7778424971332813e-06,
+      "loss": 0.7904,
+      "step": 1900
+    },
+    {
+      "epoch": 0.7354639969195226,
+      "grad_norm": 0.2049545794725418,
+      "learning_rate": 1.7304115736010407e-06,
+      "loss": 0.7896,
+      "step": 1910
+    },
+    {
+      "epoch": 0.7393145937620331,
+      "grad_norm": 0.2120913416147232,
+      "learning_rate": 1.6834891968017108e-06,
+      "loss": 0.7753,
+      "step": 1920
+    },
+    {
+      "epoch": 0.7431651906045437,
+      "grad_norm": 0.22707639634609222,
+      "learning_rate": 1.6370826649692884e-06,
+      "loss": 0.7978,
+      "step": 1930
+    },
+    {
+      "epoch": 0.7470157874470543,
+      "grad_norm": 0.19694669544696808,
+      "learning_rate": 1.5911991961040463e-06,
+      "loss": 0.7934,
+      "step": 1940
+    },
+    {
+      "epoch": 0.7508663842895649,
+      "grad_norm": 0.2179563343524933,
+      "learning_rate": 1.545845926849851e-06,
+      "loss": 0.7734,
+      "step": 1950
+    },
+    {
+      "epoch": 0.7547169811320755,
+      "grad_norm": 0.2215605080127716,
+      "learning_rate": 1.5010299113841397e-06,
+      "loss": 0.8028,
+      "step": 1960
+    },
+    {
+      "epoch": 0.7585675779745861,
+      "grad_norm": 0.22168651223182678,
+      "learning_rate": 1.4567581203207238e-06,
+      "loss": 0.7905,
+      "step": 1970
+    },
+    {
+      "epoch": 0.7624181748170966,
+      "grad_norm": 0.22104042768478394,
+      "learning_rate": 1.413037439625593e-06,
+      "loss": 0.7903,
+      "step": 1980
+    },
+    {
+      "epoch": 0.7662687716596073,
+      "grad_norm": 0.24376706779003143,
+      "learning_rate": 1.3698746695458847e-06,
+      "loss": 0.7773,
+      "step": 1990
+    },
+    {
+      "epoch": 0.7701193685021178,
+      "grad_norm": 0.20787647366523743,
+      "learning_rate": 1.3272765235521745e-06,
+      "loss": 0.7798,
+      "step": 2000
+    },
+    {
+      "epoch": 0.7739699653446284,
+      "grad_norm": 0.19173748791217804,
+      "learning_rate": 1.2852496272942877e-06,
+      "loss": 0.7857,
+      "step": 2010
+    },
+    {
+      "epoch": 0.777820562187139,
+      "grad_norm": 0.2004566490650177,
+      "learning_rate": 1.2438005175707396e-06,
+      "loss": 0.7691,
+      "step": 2020
+    },
+    {
+      "epoch": 0.7816711590296496,
+      "grad_norm": 0.20008961856365204,
+      "learning_rate": 1.2029356413120297e-06,
+      "loss": 0.7641,
+      "step": 2030
+    },
+    {
+      "epoch": 0.7855217558721602,
+      "grad_norm": 0.209969624876976,
+      "learning_rate": 1.1626613545778786e-06,
+      "loss": 0.7798,
+      "step": 2040
+    },
+    {
+      "epoch": 0.7893723527146708,
+      "grad_norm": 0.20725101232528687,
+      "learning_rate": 1.1229839215686294e-06,
+      "loss": 0.7814,
+      "step": 2050
+    },
+    {
+      "epoch": 0.7932229495571813,
+      "grad_norm": 0.19838294386863708,
+      "learning_rate": 1.0839095136509188e-06,
+      "loss": 0.7685,
+      "step": 2060
+    },
+    {
+      "epoch": 0.797073546399692,
+      "grad_norm": 0.21706055104732513,
+      "learning_rate": 1.045444208397791e-06,
+      "loss": 0.7859,
+      "step": 2070
+    },
+    {
+      "epoch": 0.8009241432422025,
+      "grad_norm": 0.19959662854671478,
+      "learning_rate": 1.007593988643402e-06,
+      "loss": 0.7611,
+      "step": 2080
+    },
+    {
+      "epoch": 0.8047747400847132,
+      "grad_norm": 0.19990290701389313,
+      "learning_rate": 9.703647415524613e-07,
+      "loss": 0.7872,
+      "step": 2090
+    },
+    {
+      "epoch": 0.8086253369272237,
+      "grad_norm": 0.21532213687896729,
+      "learning_rate": 9.337622577045485e-07,
+      "loss": 0.7817,
+      "step": 2100
+    },
+    {
+      "epoch": 0.8124759337697343,
+      "grad_norm": 0.19992892444133759,
+      "learning_rate": 8.977922301934588e-07,
+      "loss": 0.781,
+      "step": 2110
+    },
+    {
+      "epoch": 0.8163265306122449,
+      "grad_norm": 0.22376137971878052,
+      "learning_rate": 8.624602537417009e-07,
+      "loss": 0.7955,
+      "step": 2120
+    },
+    {
+      "epoch": 0.8201771274547555,
+      "grad_norm": 0.21873505413532257,
+      "learning_rate": 8.277718238303123e-07,
+      "loss": 0.7782,
+      "step": 2130
+    },
+    {
+      "epoch": 0.8240277242972661,
+      "grad_norm": 0.20839239656925201,
+      "learning_rate": 7.937323358440935e-07,
+      "loss": 0.7797,
+      "step": 2140
+    },
+    {
+      "epoch": 0.8278783211397767,
+      "grad_norm": 0.21866518259048462,
+      "learning_rate": 7.603470842324223e-07,
+      "loss": 0.7699,
+      "step": 2150
+    },
+    {
+      "epoch": 0.8317289179822872,
+      "grad_norm": 0.20118151605129242,
+      "learning_rate": 7.276212616857575e-07,
+      "loss": 0.7907,
+      "step": 2160
+    },
+    {
+      "epoch": 0.8355795148247979,
+      "grad_norm": 0.2041211575269699,
+      "learning_rate": 6.955599583279837e-07,
+      "loss": 0.7849,
+      "step": 2170
+    },
+    {
+      "epoch": 0.8394301116673084,
+      "grad_norm": 0.1908172070980072,
+      "learning_rate": 6.641681609246981e-07,
+      "loss": 0.7782,
+      "step": 2180
+    },
+    {
+      "epoch": 0.8432807085098191,
+      "grad_norm": 0.21748055517673492,
+      "learning_rate": 6.334507521075711e-07,
+      "loss": 0.7828,
+      "step": 2190
+    },
+    {
+      "epoch": 0.8471313053523296,
+      "grad_norm": 0.21843403577804565,
+      "learning_rate": 6.034125096149179e-07,
+      "loss": 0.7825,
+      "step": 2200
+    },
+    {
+      "epoch": 0.8509819021948402,
+      "grad_norm": 0.20849968492984772,
+      "learning_rate": 5.740581055485722e-07,
+      "loss": 0.7536,
+      "step": 2210
+    },
+    {
+      "epoch": 0.8548324990373508,
+      "grad_norm": 0.20477400720119476,
+      "learning_rate": 5.453921056471961e-07,
+      "loss": 0.7879,
+      "step": 2220
+    },
+    {
+      "epoch": 0.8586830958798614,
+      "grad_norm": 0.21091175079345703,
+      "learning_rate": 5.174189685761288e-07,
+      "loss": 0.7803,
+      "step": 2230
+    },
+    {
+      "epoch": 0.862533692722372,
+      "grad_norm": 0.20409975945949554,
+      "learning_rate": 4.901430452338979e-07,
+      "loss": 0.7903,
+      "step": 2240
+    },
+    {
+      "epoch": 0.8663842895648826,
+      "grad_norm": 0.20046408474445343,
+      "learning_rate": 4.6356857807548286e-07,
+      "loss": 0.7798,
+      "step": 2250
+    },
+    {
+      "epoch": 0.8702348864073931,
+      "grad_norm": 0.23115174472332,
+      "learning_rate": 4.376997004524541e-07,
+      "loss": 0.7613,
+      "step": 2260
+    },
+    {
+      "epoch": 0.8740854832499038,
+      "grad_norm": 0.20057055354118347,
+      "learning_rate": 4.12540435970073e-07,
+      "loss": 0.7816,
+      "step": 2270
+    },
+    {
+      "epoch": 0.8779360800924143,
+      "grad_norm": 0.19855555891990662,
+      "learning_rate": 3.880946978614703e-07,
+      "loss": 0.7834,
+      "step": 2280
+    },
+    {
+      "epoch": 0.881786676934925,
+      "grad_norm": 0.20400922000408173,
+      "learning_rate": 3.643662883789878e-07,
+      "loss": 0.7917,
+      "step": 2290
+    },
+    {
+      "epoch": 0.8856372737774355,
+      "grad_norm": 0.19021792709827423,
+      "learning_rate": 3.41358898202781e-07,
+      "loss": 0.7712,
+      "step": 2300
+    },
+    {
+      "epoch": 0.889487870619946,
+      "grad_norm": 0.19559277594089508,
+      "learning_rate": 3.190761058667741e-07,
+      "loss": 0.7814,
+      "step": 2310
+    },
+    {
+      "epoch": 0.8933384674624567,
+      "grad_norm": 0.18431700766086578,
+      "learning_rate": 2.9752137720206463e-07,
+      "loss": 0.7856,
+      "step": 2320
+    },
+    {
+      "epoch": 0.8971890643049673,
+      "grad_norm": 0.2083376795053482,
+      "learning_rate": 2.7669806479785067e-07,
+      "loss": 0.7951,
+      "step": 2330
+    },
+    {
+      "epoch": 0.9010396611474779,
+      "grad_norm": 0.19514045119285583,
+      "learning_rate": 2.566094074799763e-07,
+      "loss": 0.7872,
+      "step": 2340
+    },
+    {
+      "epoch": 0.9048902579899885,
+      "grad_norm": 0.20064011216163635,
+      "learning_rate": 2.3725852980716557e-07,
+      "loss": 0.7654,
+      "step": 2350
+    },
+    {
+      "epoch": 0.908740854832499,
+      "grad_norm": 0.19932985305786133,
+      "learning_rate": 2.1864844158503983e-07,
+      "loss": 0.7806,
+      "step": 2360
+    },
+    {
+      "epoch": 0.9125914516750097,
+      "grad_norm": 0.22797022759914398,
+      "learning_rate": 2.007820373979702e-07,
+      "loss": 0.7746,
+      "step": 2370
+    },
+    {
+      "epoch": 0.9164420485175202,
+      "grad_norm": 0.21179082989692688,
+      "learning_rate": 1.8366209615886532e-07,
+      "loss": 0.7834,
+      "step": 2380
+    },
+    {
+      "epoch": 0.9202926453600309,
+      "grad_norm": 0.1975056678056717,
+      "learning_rate": 1.6729128067693613e-07,
+      "loss": 0.7944,
+      "step": 2390
+    },
+    {
+      "epoch": 0.9241432422025414,
+      "grad_norm": 0.21489909291267395,
+      "learning_rate": 1.5167213724353426e-07,
+      "loss": 0.7893,
+      "step": 2400
+    },
+    {
+      "epoch": 0.927993839045052,
+      "grad_norm": 0.20665891468524933,
+      "learning_rate": 1.3680709523610058e-07,
+      "loss": 0.7675,
+      "step": 2410
+    },
+    {
+      "epoch": 0.9318444358875626,
+      "grad_norm": 0.19859078526496887,
+      "learning_rate": 1.2269846674030418e-07,
+      "loss": 0.799,
+      "step": 2420
+    },
+    {
+      "epoch": 0.9356950327300732,
+      "grad_norm": 0.2096828669309616,
+      "learning_rate": 1.0934844619042595e-07,
+      "loss": 0.7905,
+      "step": 2430
+    },
+    {
+      "epoch": 0.9395456295725837,
+      "grad_norm": 0.21714802086353302,
+      "learning_rate": 9.675911002803961e-08,
+      "loss": 0.7835,
+      "step": 2440
+    },
+    {
+      "epoch": 0.9433962264150944,
+      "grad_norm": 0.22199806571006775,
+      "learning_rate": 8.493241637904381e-08,
+      "loss": 0.7709,
+      "step": 2450
+    },
+    {
+      "epoch": 0.9472468232576049,
+      "grad_norm": 0.21588142216205597,
+      "learning_rate": 7.387020474909689e-08,
+      "loss": 0.7971,
+      "step": 2460
+    },
+    {
+      "epoch": 0.9510974201001156,
+      "grad_norm": 0.2395700216293335,
+      "learning_rate": 6.357419573750856e-08,
+      "loss": 0.793,
+      "step": 2470
+    },
+    {
+      "epoch": 0.9549480169426261,
+      "grad_norm": 0.21236297488212585,
+      "learning_rate": 5.4045990769613365e-08,
+      "loss": 0.7793,
+      "step": 2480
+    },
+    {
+      "epoch": 0.9587986137851366,
+      "grad_norm": 0.2261745184659958,
+      "learning_rate": 4.5287071847694434e-08,
+      "loss": 0.7795,
+      "step": 2490
+    },
+    {
+      "epoch": 0.9626492106276473,
+      "grad_norm": 0.20829255878925323,
+      "learning_rate": 3.7298801320467817e-08,
+      "loss": 0.7802,
+      "step": 2500
+    },
+    {
+      "epoch": 0.9664998074701578,
+      "grad_norm": 0.20765310525894165,
+      "learning_rate": 3.0082421671192576e-08,
+      "loss": 0.7791,
+      "step": 2510
+    },
+    {
+      "epoch": 0.9703504043126685,
+      "grad_norm": 0.20846140384674072,
+      "learning_rate": 2.3639055324410355e-08,
+      "loss": 0.7927,
+      "step": 2520
+    },
+    {
+      "epoch": 0.974201001155179,
+      "grad_norm": 0.20417754352092743,
+      "learning_rate": 1.7969704471369477e-08,
+      "loss": 0.7754,
+      "step": 2530
+    },
+    {
+      "epoch": 0.9780515979976896,
+      "grad_norm": 0.19036881625652313,
+      "learning_rate": 1.3075250914142413e-08,
+      "loss": 0.7737,
+      "step": 2540
+    },
+    {
+      "epoch": 0.9819021948402002,
+      "grad_norm": 0.19493336975574493,
+      "learning_rate": 8.956455928476049e-09,
+      "loss": 0.7909,
+      "step": 2550
+    },
+    {
+      "epoch": 0.9857527916827108,
+      "grad_norm": 0.20877589285373688,
+      "learning_rate": 5.6139601453786275e-09,
+      "loss": 0.7767,
+      "step": 2560
+    },
+    {
+      "epoch": 0.9896033885252215,
+      "grad_norm": 0.20326198637485504,
+      "learning_rate": 3.0482834514805602e-09,
+      "loss": 0.7717,
+      "step": 2570
+    },
+    {
+      "epoch": 0.993453985367732,
+      "grad_norm": 0.1961904764175415,
+      "learning_rate": 1.2598249081713366e-09,
+      "loss": 0.7633,
+      "step": 2580
+    },
+    {
+      "epoch": 0.9973045822102425,
+      "grad_norm": 0.21112486720085144,
+      "learning_rate": 2.488626895302826e-10,
+      "loss": 0.7932,
+      "step": 2590
+    },
+    {
+      "epoch": 1.0,
+      "step": 2597,
+      "total_flos": 5.970141653975958e+18,
+      "train_loss": 0.795934819973199,
+      "train_runtime": 16842.5027,
+      "train_samples_per_second": 39.478,
+      "train_steps_per_second": 0.154
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2597,
+  "num_input_tokens_seen": 584317136,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.970141653975958e+18,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

continuous/smollm2/dino_vitb8/finetune/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4eecf7fd4f6e763d01dbe226a53fca5b3e294fc1b6330e222d7d74a797c5e270
+size 6264

continuous/smollm2/dino_vitb8/pretrain/config.json

@@ -0,0 +1,70 @@
+{
+  "add_faster_video": false,
+  "add_time_instruction": false,
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "faster_token_stride": 10,
+  "force_sample": false,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_aspect_ratio": "square",
+  "image_crop_resolution": null,
+  "image_grid_pinpoints": null,
+  "image_split_resolution": null,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mm_hidden_size": 768,
+  "mm_newline_position": "grid",
+  "mm_patch_merge_type": "flat",
+  "mm_projector_lr": null,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_resampler_type": null,
+  "mm_spatial_pool_mode": "bilinear",
+  "mm_spatial_pool_stride": null,
+  "mm_tunable_parts": "mm_mlp_adapter",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -1,
+  "mm_vision_tower": "hf:/cache/ckpt/download/tokenizer/continuous/dino/dino-vitb8",
+  "mm_vision_tower_lr": null,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "pos_skipping_range": 4096,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 130000,
+  "tie_word_embeddings": true,
+  "tokenizer_model_max_length": 8192,
+  "tokenizer_padding_side": "right",
+  "transformers.js_config": {
+    "dtype": "q4",
+    "kv_cache_dtype": {
+      "fp16": "float16",
+      "q4f16": "float16"
+    },
+    "use_external_data_format": {
+      "model.onnx": true,
+      "model_fp16.onnx": true
+    }
+  },
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "use_mm_proj": true,
+  "use_pos_skipping": false,
+  "vision_tower_pretrained": null,
+  "vocab_size": 49152
+}

continuous/smollm2/dino_vitb8/pretrain/mm_projector.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff32d6d37002d85b07bc22ab3552197db2ff5e201bdc1fb1bf15a9ba7fca488b
+size 23087160

continuous/smollm2/dino_vitb8/pretrain/trainer_state.json

@@ -0,0 +1,421 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 545,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01834862385321101,
+      "grad_norm": 3.2308952808380127,
+      "learning_rate": 0.00032142857142857147,
+      "loss": 3.8152,
+      "step": 10
+    },
+    {
+      "epoch": 0.03669724770642202,
+      "grad_norm": 0.5146101713180542,
+      "learning_rate": 0.0006785714285714287,
+      "loss": 3.153,
+      "step": 20
+    },
+    {
+      "epoch": 0.05504587155963303,
+      "grad_norm": 0.06879637390375137,
+      "learning_rate": 0.0009999907688176173,
+      "loss": 2.9511,
+      "step": 30
+    },
+    {
+      "epoch": 0.07339449541284404,
+      "grad_norm": 0.03756636753678322,
+      "learning_rate": 0.0009988834393115766,
+      "loss": 2.842,
+      "step": 40
+    },
+    {
+      "epoch": 0.09174311926605505,
+      "grad_norm": 0.08637754619121552,
+      "learning_rate": 0.0009959345572940772,
+      "loss": 2.8091,
+      "step": 50
+    },
+    {
+      "epoch": 0.11009174311926606,
+      "grad_norm": 0.09695123881101608,
+      "learning_rate": 0.0009911550081155983,
+      "loss": 2.7786,
+      "step": 60
+    },
+    {
+      "epoch": 0.12844036697247707,
+      "grad_norm": 0.0454535037279129,
+      "learning_rate": 0.0009845624347566433,
+      "loss": 2.758,
+      "step": 70
+    },
+    {
+      "epoch": 0.14678899082568808,
+      "grad_norm": 0.08782809972763062,
+      "learning_rate": 0.0009761811727013547,
+      "loss": 2.7415,
+      "step": 80
+    },
+    {
+      "epoch": 0.1651376146788991,
+      "grad_norm": 0.08510341495275497,
+      "learning_rate": 0.0009660421601067666,
+      "loss": 2.7207,
+      "step": 90
+    },
+    {
+      "epoch": 0.1834862385321101,
+      "grad_norm": 0.07792966812849045,
+      "learning_rate": 0.0009541828235992926,
+      "loss": 2.7146,
+      "step": 100
+    },
+    {
+      "epoch": 0.2018348623853211,
+      "grad_norm": 0.09817753732204437,
+      "learning_rate": 0.0009406469401200151,
+      "loss": 2.6919,
+      "step": 110
+    },
+    {
+      "epoch": 0.22018348623853212,
+      "grad_norm": 0.17000900208950043,
+      "learning_rate": 0.0009254844753287492,
+      "loss": 2.685,
+      "step": 120
+    },
+    {
+      "epoch": 0.23853211009174313,
+      "grad_norm": 0.05132520571351051,
+      "learning_rate": 0.0009087513991633924,
+      "loss": 2.6676,
+      "step": 130
+    },
+    {
+      "epoch": 0.25688073394495414,
+      "grad_norm": 0.0749715194106102,
+      "learning_rate": 0.0008905094792353917,
+      "loss": 2.6524,
+      "step": 140
+    },
+    {
+      "epoch": 0.27522935779816515,
+      "grad_norm": 0.11878398805856705,
+      "learning_rate": 0.0008708260528239789,
+      "loss": 2.6547,
+      "step": 150
+    },
+    {
+      "epoch": 0.29357798165137616,
+      "grad_norm": 0.09599041938781738,
+      "learning_rate": 0.0008497737783108237,
+      "loss": 2.6438,
+      "step": 160
+    },
+    {
+      "epoch": 0.3119266055045872,
+      "grad_norm": 0.08247403800487518,
+      "learning_rate": 0.0008274303669726426,
+      "loss": 2.6394,
+      "step": 170
+    },
+    {
+      "epoch": 0.3302752293577982,
+      "grad_norm": 0.06197255477309227,
+      "learning_rate": 0.0008038782961218137,
+      "loss": 2.6262,
+      "step": 180
+    },
+    {
+      "epoch": 0.3486238532110092,
+      "grad_norm": 0.07590397447347641,
+      "learning_rate": 0.000779204504653894,
+      "loss": 2.6245,
+      "step": 190
+    },
+    {
+      "epoch": 0.3669724770642202,
+      "grad_norm": 0.06055331230163574,
+      "learning_rate": 0.0007535000721258791,
+      "loss": 2.6018,
+      "step": 200
+    },
+    {
+      "epoch": 0.3853211009174312,
+      "grad_norm": 0.11831152439117432,
+      "learning_rate": 0.00072685988254984,
+      "loss": 2.6026,
+      "step": 210
+    },
+    {
+      "epoch": 0.4036697247706422,
+      "grad_norm": 0.10891497880220413,
+      "learning_rate": 0.0006993822741429906,
+      "loss": 2.61,
+      "step": 220
+    },
+    {
+      "epoch": 0.42201834862385323,
+      "grad_norm": 0.08018132299184799,
+      "learning_rate": 0.0006711686763270818,
+      "loss": 2.6036,
+      "step": 230
+    },
+    {
+      "epoch": 0.44036697247706424,
+      "grad_norm": 0.07834528386592865,
+      "learning_rate": 0.0006423232353170831,
+      "loss": 2.5968,
+      "step": 240
+    },
+    {
+      "epoch": 0.45871559633027525,
+      "grad_norm": 0.07757870852947235,
+      "learning_rate": 0.0006129524296812335,
+      "loss": 2.5882,
+      "step": 250
+    },
+    {
+      "epoch": 0.47706422018348627,
+      "grad_norm": 0.06161823868751526,
+      "learning_rate": 0.0005831646772915651,
+      "loss": 2.5959,
+      "step": 260
+    },
+    {
+      "epoch": 0.4954128440366973,
+      "grad_norm": 0.06286874413490295,
+      "learning_rate": 0.0005530699351157809,
+      "loss": 2.5736,
+      "step": 270
+    },
+    {
+      "epoch": 0.5137614678899083,
+      "grad_norm": 0.07163397967815399,
+      "learning_rate": 0.0005227792933277943,
+      "loss": 2.5787,
+      "step": 280
+    },
+    {
+      "epoch": 0.5321100917431193,
+      "grad_norm": 0.06091686710715294,
+      "learning_rate": 0.0004924045652352092,
+      "loss": 2.5619,
+      "step": 290
+    },
+    {
+      "epoch": 0.5504587155963303,
+      "grad_norm": 0.061662547290325165,
+      "learning_rate": 0.0004620578745374633,
+      "loss": 2.5635,
+      "step": 300
+    },
+    {
+      "epoch": 0.5688073394495413,
+      "grad_norm": 0.07054687291383743,
+      "learning_rate": 0.0004318512414382059,
+      "loss": 2.569,
+      "step": 310
+    },
+    {
+      "epoch": 0.5871559633027523,
+      "grad_norm": 0.06268132477998734,
+      "learning_rate": 0.0004018961691397148,
+      "loss": 2.5715,
+      "step": 320
+    },
+    {
+      "epoch": 0.6055045871559633,
+      "grad_norm": 0.060098759829998016,
+      "learning_rate": 0.0003723032322457458,
+      "loss": 2.5532,
+      "step": 330
+    },
+    {
+      "epoch": 0.6238532110091743,
+      "grad_norm": 0.080913245677948,
+      "learning_rate": 0.0003431816685921625,
+      "loss": 2.5582,
+      "step": 340
+    },
+    {
+      "epoch": 0.6422018348623854,
+      "grad_norm": 0.05525069683790207,
+      "learning_rate": 0.0003146389760120416,
+      "loss": 2.561,
+      "step": 350
+    },
+    {
+      "epoch": 0.6605504587155964,
+      "grad_norm": 0.06564752012491226,
+      "learning_rate": 0.00028678051552373485,
+      "loss": 2.5628,
+      "step": 360
+    },
+    {
+      "epoch": 0.6788990825688074,
+      "grad_norm": 0.06695030629634857,
+      "learning_rate": 0.0002597091224066581,
+      "loss": 2.5531,
+      "step": 370
+    },
+    {
+      "epoch": 0.6972477064220184,
+      "grad_norm": 0.05037560313940048,
+      "learning_rate": 0.0002335247266004629,
+      "loss": 2.562,
+      "step": 380
+    },
+    {
+      "epoch": 0.7155963302752294,
+      "grad_norm": 0.06829845160245895,
+      "learning_rate": 0.000208323983828828,
+      "loss": 2.5529,
+      "step": 390
+    },
+    {
+      "epoch": 0.7339449541284404,
+      "grad_norm": 0.05935126170516014,
+      "learning_rate": 0.0001841999188095224,
+      "loss": 2.5494,
+      "step": 400
+    },
+    {
+      "epoch": 0.7522935779816514,
+      "grad_norm": 0.052902352064847946,
+      "learning_rate": 0.00016124158186777676,
+      "loss": 2.5449,
+      "step": 410
+    },
+    {
+      "epoch": 0.7706422018348624,
+      "grad_norm": 0.05345124378800392,
+      "learning_rate": 0.0001395337202205211,
+      "loss": 2.5406,
+      "step": 420
+    },
+    {
+      "epoch": 0.7889908256880734,
+      "grad_norm": 0.054040826857089996,
+      "learning_rate": 0.00011915646514489292,
+      "loss": 2.5443,
+      "step": 430
+    },
+    {
+      "epoch": 0.8073394495412844,
+      "grad_norm": 0.06634188443422318,
+      "learning_rate": 0.00010018503618578818,
+      "loss": 2.5509,
+      "step": 440
+    },
+    {
+      "epoch": 0.8256880733944955,
+      "grad_norm": 0.04963618889451027,
+      "learning_rate": 8.268946349432583e-05,
+      "loss": 2.5483,
+      "step": 450
+    },
+    {
+      "epoch": 0.8440366972477065,
+      "grad_norm": 0.052710969001054764,
+      "learning_rate": 6.673432932217117e-05,
+      "loss": 2.5379,
+      "step": 460
+    },
+    {
+      "epoch": 0.8623853211009175,
+      "grad_norm": 0.05344215780496597,
+      "learning_rate": 5.2378529625954684e-05,
+      "loss": 2.5299,
+      "step": 470
+    },
+    {
+      "epoch": 0.8807339449541285,
+      "grad_norm": 0.04377128928899765,
+      "learning_rate": 3.9675056661785556e-05,
+      "loss": 2.5363,
+      "step": 480
+    },
+    {
+      "epoch": 0.8990825688073395,
+      "grad_norm": 0.04370250180363655,
+      "learning_rate": 2.8670803372379538e-05,
+      "loss": 2.541,
+      "step": 490
+    },
+    {
+      "epoch": 0.9174311926605505,
+      "grad_norm": 0.043901074677705765,
+      "learning_rate": 1.9406390288876586e-05,
+      "loss": 2.5429,
+      "step": 500
+    },
+    {
+      "epoch": 0.9357798165137615,
+      "grad_norm": 0.05017153546214104,
+      "learning_rate": 1.191601558631461e-05,
+      "loss": 2.5344,
+      "step": 510
+    },
+    {
+      "epoch": 0.9541284403669725,
+      "grad_norm": 0.044320981949567795,
+      "learning_rate": 6.2273288462559305e-06,
+      "loss": 2.5302,
+      "step": 520
+    },
+    {
+      "epoch": 0.9724770642201835,
+      "grad_norm": 0.04386711120605469,
+      "learning_rate": 2.361328992552314e-06,
+      "loss": 2.5328,
+      "step": 530
+    },
+    {
+      "epoch": 0.9908256880733946,
+      "grad_norm": 0.04582589119672775,
+      "learning_rate": 3.322867770044202e-07,
+      "loss": 2.537,
+      "step": 540
+    },
+    {
+      "epoch": 1.0,
+      "step": 545,
+      "total_flos": 3.398158394589184e+17,
+      "train_loss": 2.6432439856572985,
+      "train_runtime": 3657.2745,
+      "train_samples_per_second": 152.608,
+      "train_steps_per_second": 0.149
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 545,
+  "num_input_tokens_seen": 33258880,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.398158394589184e+17,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

continuous/smollm2/dino_vits16/finetune/chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

continuous/smollm2/dino_vits16/finetune/config.json

@@ -0,0 +1,70 @@
+{
+  "add_faster_video": false,
+  "add_time_instruction": false,
+  "architectures": [
+    "LlavaLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "faster_token_stride": 10,
+  "force_sample": false,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_aspect_ratio": "square",
+  "image_crop_resolution": null,
+  "image_grid_pinpoints": null,
+  "image_split_resolution": null,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mm_hidden_size": 384,
+  "mm_newline_position": "grid",
+  "mm_patch_merge_type": "flat",
+  "mm_projector_lr": 0.0001,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_resampler_type": null,
+  "mm_spatial_pool_mode": "bilinear",
+  "mm_spatial_pool_stride": null,
+  "mm_tunable_parts": "mm_mlp_adapter,mm_language_model",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -1,
+  "mm_vision_tower": "hf:/cache/ckpt/download/tokenizer/continuous/dino/dino-vits16",
+  "mm_vision_tower_lr": null,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "pos_skipping_range": 4096,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 130000,
+  "tie_word_embeddings": true,
+  "tokenizer_model_max_length": 8192,
+  "tokenizer_padding_side": "right",
+  "transformers.js_config": {
+    "dtype": "q4",
+    "kv_cache_dtype": {
+      "fp16": "float16",
+      "q4f16": "float16"
+    },
+    "use_external_data_format": {
+      "model.onnx": true,
+      "model_fp16.onnx": true
+    }
+  },
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "use_mm_proj": true,
+  "use_pos_skipping": false,
+  "vision_tower_pretrained": null,
+  "vocab_size": 49152
+}

continuous/smollm2/dino_vits16/finetune/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    2
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.57.6"
+}

continuous/smollm2/dino_vits16/finetune/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c897bb183fe0f79312e1b59226d9aceb7a8fb46459d9cc518d09b64fd7e9ca1b
+size 3687700928

continuous/smollm2/dino_vits16/finetune/special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

continuous/smollm2/dino_vits16/finetune/tokenizer_config.json

@@ -0,0 +1,155 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

continuous/smollm2/dino_vits16/finetune/trainer_state.json

@@ -0,0 +1,1856 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 2597,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.003850596842510589,
+      "grad_norm": 3.4574599266052246,
+      "learning_rate": 1.153846153846154e-06,
+      "loss": 1.1913,
+      "step": 10
+    },
+    {
+      "epoch": 0.007701193685021178,
+      "grad_norm": 2.096057891845703,
+      "learning_rate": 2.435897435897436e-06,
+      "loss": 1.1025,
+      "step": 20
+    },
+    {
+      "epoch": 0.011551790527531768,
+      "grad_norm": 2.001840591430664,
+      "learning_rate": 3.7179487179487184e-06,
+      "loss": 1.0273,
+      "step": 30
+    },
+    {
+      "epoch": 0.015402387370042356,
+      "grad_norm": 1.3791970014572144,
+      "learning_rate": 5e-06,
+      "loss": 0.9896,
+      "step": 40
+    },
+    {
+      "epoch": 0.019252984212552945,
+      "grad_norm": 1.2959558963775635,
+      "learning_rate": 6.282051282051282e-06,
+      "loss": 0.9792,
+      "step": 50
+    },
+    {
+      "epoch": 0.023103581055063535,
+      "grad_norm": 0.3704852759838104,
+      "learning_rate": 7.564102564102564e-06,
+      "loss": 0.9138,
+      "step": 60
+    },
+    {
+      "epoch": 0.026954177897574125,
+      "grad_norm": 0.30978262424468994,
+      "learning_rate": 8.846153846153847e-06,
+      "loss": 0.8667,
+      "step": 70
+    },
+    {
+      "epoch": 0.030804774740084712,
+      "grad_norm": 0.3048102855682373,
+      "learning_rate": 9.999996111488723e-06,
+      "loss": 0.8819,
+      "step": 80
+    },
+    {
+      "epoch": 0.0346553715825953,
+      "grad_norm": 0.30099010467529297,
+      "learning_rate": 9.999529497453782e-06,
+      "loss": 0.8683,
+      "step": 90
+    },
+    {
+      "epoch": 0.03850596842510589,
+      "grad_norm": 0.24671384692192078,
+      "learning_rate": 9.998285264324193e-06,
+      "loss": 0.8392,
+      "step": 100
+    },
+    {
+      "epoch": 0.04235656526761648,
+      "grad_norm": 0.34807512164115906,
+      "learning_rate": 9.996263605626052e-06,
+      "loss": 0.8294,
+      "step": 110
+    },
+    {
+      "epoch": 0.04620716211012707,
+      "grad_norm": 0.2726980149745941,
+      "learning_rate": 9.99346483580503e-06,
+      "loss": 0.8502,
+      "step": 120
+    },
+    {
+      "epoch": 0.05005775895263766,
+      "grad_norm": 0.27817705273628235,
+      "learning_rate": 9.989889390177467e-06,
+      "loss": 0.8269,
+      "step": 130
+    },
+    {
+      "epoch": 0.05390835579514825,
+      "grad_norm": 0.3018190264701843,
+      "learning_rate": 9.985537824862647e-06,
+      "loss": 0.8397,
+      "step": 140
+    },
+    {
+      "epoch": 0.05775895263765884,
+      "grad_norm": 0.2674808204174042,
+      "learning_rate": 9.980410816696317e-06,
+      "loss": 0.8175,
+      "step": 150
+    },
+    {
+      "epoch": 0.061609549480169425,
+      "grad_norm": 0.321941614151001,
+      "learning_rate": 9.97450916312541e-06,
+      "loss": 0.8221,
+      "step": 160
+    },
+    {
+      "epoch": 0.06546014632268002,
+      "grad_norm": 0.24123156070709229,
+      "learning_rate": 9.967833782084004e-06,
+      "loss": 0.8228,
+      "step": 170
+    },
+    {
+      "epoch": 0.0693107431651906,
+      "grad_norm": 0.24453729391098022,
+      "learning_rate": 9.960385711850554e-06,
+      "loss": 0.8552,
+      "step": 180
+    },
+    {
+      "epoch": 0.07316134000770119,
+      "grad_norm": 0.23953835666179657,
+      "learning_rate": 9.952166110886394e-06,
+      "loss": 0.8423,
+      "step": 190
+    },
+    {
+      "epoch": 0.07701193685021178,
+      "grad_norm": 0.24519112706184387,
+      "learning_rate": 9.943176257655567e-06,
+      "loss": 0.8414,
+      "step": 200
+    },
+    {
+      "epoch": 0.08086253369272237,
+      "grad_norm": 0.24229593575000763,
+      "learning_rate": 9.933417550425952e-06,
+      "loss": 0.8264,
+      "step": 210
+    },
+    {
+      "epoch": 0.08471313053523297,
+      "grad_norm": 0.2858675718307495,
+      "learning_rate": 9.9228915070518e-06,
+      "loss": 0.8195,
+      "step": 220
+    },
+    {
+      "epoch": 0.08856372737774355,
+      "grad_norm": 0.23091387748718262,
+      "learning_rate": 9.911599764737632e-06,
+      "loss": 0.8149,
+      "step": 230
+    },
+    {
+      "epoch": 0.09241432422025414,
+      "grad_norm": 0.25979727506637573,
+      "learning_rate": 9.899544079783606e-06,
+      "loss": 0.8078,
+      "step": 240
+    },
+    {
+      "epoch": 0.09626492106276473,
+      "grad_norm": 0.26150432229042053,
+      "learning_rate": 9.886726327312338e-06,
+      "loss": 0.8261,
+      "step": 250
+    },
+    {
+      "epoch": 0.10011551790527531,
+      "grad_norm": 0.25931820273399353,
+      "learning_rate": 9.87314850097724e-06,
+      "loss": 0.8253,
+      "step": 260
+    },
+    {
+      "epoch": 0.1039661147477859,
+      "grad_norm": 0.2280590683221817,
+      "learning_rate": 9.858812712652452e-06,
+      "loss": 0.8056,
+      "step": 270
+    },
+    {
+      "epoch": 0.1078167115902965,
+      "grad_norm": 0.2466902881860733,
+      "learning_rate": 9.843721192104341e-06,
+      "loss": 0.805,
+      "step": 280
+    },
+    {
+      "epoch": 0.11166730843280709,
+      "grad_norm": 0.23557673394680023,
+      "learning_rate": 9.827876286644698e-06,
+      "loss": 0.8349,
+      "step": 290
+    },
+    {
+      "epoch": 0.11551790527531768,
+      "grad_norm": 0.24220828711986542,
+      "learning_rate": 9.811280460765643e-06,
+      "loss": 0.7935,
+      "step": 300
+    },
+    {
+      "epoch": 0.11936850211782826,
+      "grad_norm": 0.2524705231189728,
+      "learning_rate": 9.793936295756292e-06,
+      "loss": 0.7993,
+      "step": 310
+    },
+    {
+      "epoch": 0.12321909896033885,
+      "grad_norm": 0.2341354787349701,
+      "learning_rate": 9.775846489301279e-06,
+      "loss": 0.8075,
+      "step": 320
+    },
+    {
+      "epoch": 0.12706969580284944,
+      "grad_norm": 0.2548820972442627,
+      "learning_rate": 9.757013855061157e-06,
+      "loss": 0.8107,
+      "step": 330
+    },
+    {
+      "epoch": 0.13092029264536004,
+      "grad_norm": 0.2219994217157364,
+      "learning_rate": 9.737441322234758e-06,
+      "loss": 0.812,
+      "step": 340
+    },
+    {
+      "epoch": 0.1347708894878706,
+      "grad_norm": 0.2445983737707138,
+      "learning_rate": 9.717131935103605e-06,
+      "loss": 0.8191,
+      "step": 350
+    },
+    {
+      "epoch": 0.1386214863303812,
+      "grad_norm": 0.21815209090709686,
+      "learning_rate": 9.696088852558398e-06,
+      "loss": 0.7986,
+      "step": 360
+    },
+    {
+      "epoch": 0.1424720831728918,
+      "grad_norm": 0.254794180393219,
+      "learning_rate": 9.674315347607686e-06,
+      "loss": 0.827,
+      "step": 370
+    },
+    {
+      "epoch": 0.14632268001540238,
+      "grad_norm": 0.2501651644706726,
+      "learning_rate": 9.651814806868796e-06,
+      "loss": 0.7923,
+      "step": 380
+    },
+    {
+      "epoch": 0.15017327685791299,
+      "grad_norm": 0.2613215446472168,
+      "learning_rate": 9.62859073004107e-06,
+      "loss": 0.7955,
+      "step": 390
+    },
+    {
+      "epoch": 0.15402387370042356,
+      "grad_norm": 0.21509675681591034,
+      "learning_rate": 9.604646729361547e-06,
+      "loss": 0.805,
+      "step": 400
+    },
+    {
+      "epoch": 0.15787447054293416,
+      "grad_norm": 0.24860499799251556,
+      "learning_rate": 9.579986529043099e-06,
+      "loss": 0.802,
+      "step": 410
+    },
+    {
+      "epoch": 0.16172506738544473,
+      "grad_norm": 0.2218461036682129,
+      "learning_rate": 9.554613964695189e-06,
+      "loss": 0.815,
+      "step": 420
+    },
+    {
+      "epoch": 0.16557566422795533,
+      "grad_norm": 0.23414246737957,
+      "learning_rate": 9.528532982727281e-06,
+      "loss": 0.7933,
+      "step": 430
+    },
+    {
+      "epoch": 0.16942626107046593,
+      "grad_norm": 0.2347807139158249,
+      "learning_rate": 9.501747639735018e-06,
+      "loss": 0.8031,
+      "step": 440
+    },
+    {
+      "epoch": 0.1732768579129765,
+      "grad_norm": 0.23189981281757355,
+      "learning_rate": 9.474262101869272e-06,
+      "loss": 0.7877,
+      "step": 450
+    },
+    {
+      "epoch": 0.1771274547554871,
+      "grad_norm": 0.26966390013694763,
+      "learning_rate": 9.446080644188141e-06,
+      "loss": 0.7968,
+      "step": 460
+    },
+    {
+      "epoch": 0.18097805159799768,
+      "grad_norm": 0.2583698034286499,
+      "learning_rate": 9.417207649992017e-06,
+      "loss": 0.8006,
+      "step": 470
+    },
+    {
+      "epoch": 0.18482864844050828,
+      "grad_norm": 0.25211766362190247,
+      "learning_rate": 9.387647610141816e-06,
+      "loss": 0.8006,
+      "step": 480
+    },
+    {
+      "epoch": 0.18867924528301888,
+      "grad_norm": 0.21609750390052795,
+      "learning_rate": 9.35740512236047e-06,
+      "loss": 0.7921,
+      "step": 490
+    },
+    {
+      "epoch": 0.19252984212552945,
+      "grad_norm": 0.20623861253261566,
+      "learning_rate": 9.326484890517809e-06,
+      "loss": 0.7941,
+      "step": 500
+    },
+    {
+      "epoch": 0.19638043896804006,
+      "grad_norm": 0.23637481033802032,
+      "learning_rate": 9.294891723898926e-06,
+      "loss": 0.7853,
+      "step": 510
+    },
+    {
+      "epoch": 0.20023103581055063,
+      "grad_norm": 0.24892392754554749,
+      "learning_rate": 9.262630536456153e-06,
+      "loss": 0.8103,
+      "step": 520
+    },
+    {
+      "epoch": 0.20408163265306123,
+      "grad_norm": 0.26681527495384216,
+      "learning_rate": 9.229706346044749e-06,
+      "loss": 0.793,
+      "step": 530
+    },
+    {
+      "epoch": 0.2079322294955718,
+      "grad_norm": 0.23376378417015076,
+      "learning_rate": 9.196124273642422e-06,
+      "loss": 0.8019,
+      "step": 540
+    },
+    {
+      "epoch": 0.2117828263380824,
+      "grad_norm": 0.20675553381443024,
+      "learning_rate": 9.161889542552842e-06,
+      "loss": 0.8065,
+      "step": 550
+    },
+    {
+      "epoch": 0.215633423180593,
+      "grad_norm": 0.23398804664611816,
+      "learning_rate": 9.127007477593193e-06,
+      "loss": 0.8106,
+      "step": 560
+    },
+    {
+      "epoch": 0.21948402002310358,
+      "grad_norm": 0.24831262230873108,
+      "learning_rate": 9.091483504265974e-06,
+      "loss": 0.829,
+      "step": 570
+    },
+    {
+      "epoch": 0.22333461686561418,
+      "grad_norm": 0.2616378664970398,
+      "learning_rate": 9.055323147915116e-06,
+      "loss": 0.7911,
+      "step": 580
+    },
+    {
+      "epoch": 0.22718521370812475,
+      "grad_norm": 0.23658575117588043,
+      "learning_rate": 9.018532032866585e-06,
+      "loss": 0.8101,
+      "step": 590
+    },
+    {
+      "epoch": 0.23103581055063535,
+      "grad_norm": 0.21037787199020386,
+      "learning_rate": 8.981115881553579e-06,
+      "loss": 0.8066,
+      "step": 600
+    },
+    {
+      "epoch": 0.23488640739314592,
+      "grad_norm": 0.2450089156627655,
+      "learning_rate": 8.943080513626476e-06,
+      "loss": 0.8248,
+      "step": 610
+    },
+    {
+      "epoch": 0.23873700423565652,
+      "grad_norm": 0.22016309201717377,
+      "learning_rate": 8.904431845047641e-06,
+      "loss": 0.802,
+      "step": 620
+    },
+    {
+      "epoch": 0.24258760107816713,
+      "grad_norm": 0.25492432713508606,
+      "learning_rate": 8.86517588717128e-06,
+      "loss": 0.8149,
+      "step": 630
+    },
+    {
+      "epoch": 0.2464381979206777,
+      "grad_norm": 0.266630619764328,
+      "learning_rate": 8.82531874580844e-06,
+      "loss": 0.809,
+      "step": 640
+    },
+    {
+      "epoch": 0.25028879476318827,
+      "grad_norm": 0.2731613516807556,
+      "learning_rate": 8.784866620277314e-06,
+      "loss": 0.7709,
+      "step": 650
+    },
+    {
+      "epoch": 0.25413939160569887,
+      "grad_norm": 0.23847374320030212,
+      "learning_rate": 8.743825802439018e-06,
+      "loss": 0.7997,
+      "step": 660
+    },
+    {
+      "epoch": 0.2579899884482095,
+      "grad_norm": 0.22277122735977173,
+      "learning_rate": 8.702202675718956e-06,
+      "loss": 0.8055,
+      "step": 670
+    },
+    {
+      "epoch": 0.2618405852907201,
+      "grad_norm": 0.2184337079524994,
+      "learning_rate": 8.660003714113959e-06,
+      "loss": 0.7897,
+      "step": 680
+    },
+    {
+      "epoch": 0.2656911821332307,
+      "grad_norm": 0.2070864588022232,
+      "learning_rate": 8.617235481185323e-06,
+      "loss": 0.8005,
+      "step": 690
+    },
+    {
+      "epoch": 0.2695417789757412,
+      "grad_norm": 0.256113737821579,
+      "learning_rate": 8.573904629037931e-06,
+      "loss": 0.8112,
+      "step": 700
+    },
+    {
+      "epoch": 0.2733923758182518,
+      "grad_norm": 0.2243078202009201,
+      "learning_rate": 8.530017897285585e-06,
+      "loss": 0.7908,
+      "step": 710
+    },
+    {
+      "epoch": 0.2772429726607624,
+      "grad_norm": 0.2336769700050354,
+      "learning_rate": 8.485582112002746e-06,
+      "loss": 0.7933,
+      "step": 720
+    },
+    {
+      "epoch": 0.281093569503273,
+      "grad_norm": 0.22662131488323212,
+      "learning_rate": 8.440604184662811e-06,
+      "loss": 0.8036,
+      "step": 730
+    },
+    {
+      "epoch": 0.2849441663457836,
+      "grad_norm": 0.25997987389564514,
+      "learning_rate": 8.395091111063118e-06,
+      "loss": 0.7894,
+      "step": 740
+    },
+    {
+      "epoch": 0.28879476318829417,
+      "grad_norm": 0.22152291238307953,
+      "learning_rate": 8.349049970236822e-06,
+      "loss": 0.7848,
+      "step": 750
+    },
+    {
+      "epoch": 0.29264536003080477,
+      "grad_norm": 0.24719084799289703,
+      "learning_rate": 8.302487923351842e-06,
+      "loss": 0.7827,
+      "step": 760
+    },
+    {
+      "epoch": 0.29649595687331537,
+      "grad_norm": 0.2542012929916382,
+      "learning_rate": 8.255412212597021e-06,
+      "loss": 0.8013,
+      "step": 770
+    },
+    {
+      "epoch": 0.30034655371582597,
+      "grad_norm": 0.24366873502731323,
+      "learning_rate": 8.207830160055688e-06,
+      "loss": 0.7948,
+      "step": 780
+    },
+    {
+      "epoch": 0.3041971505583365,
+      "grad_norm": 0.20762425661087036,
+      "learning_rate": 8.159749166566786e-06,
+      "loss": 0.7815,
+      "step": 790
+    },
+    {
+      "epoch": 0.3080477474008471,
+      "grad_norm": 0.22425539791584015,
+      "learning_rate": 8.11117671057377e-06,
+      "loss": 0.8024,
+      "step": 800
+    },
+    {
+      "epoch": 0.3118983442433577,
+      "grad_norm": 0.22715339064598083,
+      "learning_rate": 8.062120346961404e-06,
+      "loss": 0.8014,
+      "step": 810
+    },
+    {
+      "epoch": 0.3157489410858683,
+      "grad_norm": 0.21747104823589325,
+      "learning_rate": 8.012587705880705e-06,
+      "loss": 0.8,
+      "step": 820
+    },
+    {
+      "epoch": 0.3195995379283789,
+      "grad_norm": 0.22453974187374115,
+      "learning_rate": 7.962586491562146e-06,
+      "loss": 0.7852,
+      "step": 830
+    },
+    {
+      "epoch": 0.32345013477088946,
+      "grad_norm": 0.23767566680908203,
+      "learning_rate": 7.912124481117357e-06,
+      "loss": 0.7834,
+      "step": 840
+    },
+    {
+      "epoch": 0.32730073161340006,
+      "grad_norm": 0.21481387317180634,
+      "learning_rate": 7.861209523329494e-06,
+      "loss": 0.7925,
+      "step": 850
+    },
+    {
+      "epoch": 0.33115132845591067,
+      "grad_norm": 0.23789338767528534,
+      "learning_rate": 7.809849537432432e-06,
+      "loss": 0.7867,
+      "step": 860
+    },
+    {
+      "epoch": 0.33500192529842127,
+      "grad_norm": 0.22976092994213104,
+      "learning_rate": 7.758052511879035e-06,
+      "loss": 0.7893,
+      "step": 870
+    },
+    {
+      "epoch": 0.33885252214093187,
+      "grad_norm": 0.23560549318790436,
+      "learning_rate": 7.705826503098642e-06,
+      "loss": 0.7924,
+      "step": 880
+    },
+    {
+      "epoch": 0.3427031189834424,
+      "grad_norm": 0.21136963367462158,
+      "learning_rate": 7.653179634243979e-06,
+      "loss": 0.7843,
+      "step": 890
+    },
+    {
+      "epoch": 0.346553715825953,
+      "grad_norm": 0.24556320905685425,
+      "learning_rate": 7.600120093927696e-06,
+      "loss": 0.7902,
+      "step": 900
+    },
+    {
+      "epoch": 0.3504043126684636,
+      "grad_norm": 0.23785413801670074,
+      "learning_rate": 7.546656134948729e-06,
+      "loss": 0.7787,
+      "step": 910
+    },
+    {
+      "epoch": 0.3542549095109742,
+      "grad_norm": 0.20226305723190308,
+      "learning_rate": 7.492796073008661e-06,
+      "loss": 0.7833,
+      "step": 920
+    },
+    {
+      "epoch": 0.3581055063534848,
+      "grad_norm": 0.27443191409111023,
+      "learning_rate": 7.438548285418326e-06,
+      "loss": 0.7665,
+      "step": 930
+    },
+    {
+      "epoch": 0.36195610319599536,
+      "grad_norm": 0.23518522083759308,
+      "learning_rate": 7.3839212097948e-06,
+      "loss": 0.7976,
+      "step": 940
+    },
+    {
+      "epoch": 0.36580670003850596,
+      "grad_norm": 0.2307562530040741,
+      "learning_rate": 7.328923342749039e-06,
+      "loss": 0.8134,
+      "step": 950
+    },
+    {
+      "epoch": 0.36965729688101656,
+      "grad_norm": 0.2377326786518097,
+      "learning_rate": 7.273563238564319e-06,
+      "loss": 0.8089,
+      "step": 960
+    },
+    {
+      "epoch": 0.37350789372352716,
+      "grad_norm": 0.26057663559913635,
+      "learning_rate": 7.217849507865724e-06,
+      "loss": 0.7803,
+      "step": 970
+    },
+    {
+      "epoch": 0.37735849056603776,
+      "grad_norm": 0.21619148552417755,
+      "learning_rate": 7.161790816280858e-06,
+      "loss": 0.7859,
+      "step": 980
+    },
+    {
+      "epoch": 0.3812090874085483,
+      "grad_norm": 0.24792227149009705,
+      "learning_rate": 7.1053958830920125e-06,
+      "loss": 0.7777,
+      "step": 990
+    },
+    {
+      "epoch": 0.3850596842510589,
+      "grad_norm": 0.23443657159805298,
+      "learning_rate": 7.0486734798799715e-06,
+      "loss": 0.7852,
+      "step": 1000
+    },
+    {
+      "epoch": 0.3889102810935695,
+      "grad_norm": 0.21006381511688232,
+      "learning_rate": 6.991632429159708e-06,
+      "loss": 0.8049,
+      "step": 1010
+    },
+    {
+      "epoch": 0.3927608779360801,
+      "grad_norm": 0.23300904035568237,
+      "learning_rate": 6.934281603008132e-06,
+      "loss": 0.8055,
+      "step": 1020
+    },
+    {
+      "epoch": 0.39661147477859066,
+      "grad_norm": 0.20858925580978394,
+      "learning_rate": 6.876629921684143e-06,
+      "loss": 0.8021,
+      "step": 1030
+    },
+    {
+      "epoch": 0.40046207162110126,
+      "grad_norm": 0.2987418472766876,
+      "learning_rate": 6.818686352241199e-06,
+      "loss": 0.7909,
+      "step": 1040
+    },
+    {
+      "epoch": 0.40431266846361186,
+      "grad_norm": 0.21070654690265656,
+      "learning_rate": 6.760459907132573e-06,
+      "loss": 0.7867,
+      "step": 1050
+    },
+    {
+      "epoch": 0.40816326530612246,
+      "grad_norm": 0.20640230178833008,
+      "learning_rate": 6.701959642809589e-06,
+      "loss": 0.792,
+      "step": 1060
+    },
+    {
+      "epoch": 0.41201386214863306,
+      "grad_norm": 0.234266459941864,
+      "learning_rate": 6.643194658312987e-06,
+      "loss": 0.7838,
+      "step": 1070
+    },
+    {
+      "epoch": 0.4158644589911436,
+      "grad_norm": 0.21693231165409088,
+      "learning_rate": 6.584174093857676e-06,
+      "loss": 0.8092,
+      "step": 1080
+    },
+    {
+      "epoch": 0.4197150558336542,
+      "grad_norm": 0.24434316158294678,
+      "learning_rate": 6.5249071294110735e-06,
+      "loss": 0.7898,
+      "step": 1090
+    },
+    {
+      "epoch": 0.4235656526761648,
+      "grad_norm": 0.24109454452991486,
+      "learning_rate": 6.465402983265274e-06,
+      "loss": 0.8025,
+      "step": 1100
+    },
+    {
+      "epoch": 0.4274162495186754,
+      "grad_norm": 0.21492362022399902,
+      "learning_rate": 6.405670910603243e-06,
+      "loss": 0.7972,
+      "step": 1110
+    },
+    {
+      "epoch": 0.431266846361186,
+      "grad_norm": 0.3023546040058136,
+      "learning_rate": 6.345720202059291e-06,
+      "loss": 0.7939,
+      "step": 1120
+    },
+    {
+      "epoch": 0.43511744320369655,
+      "grad_norm": 0.2433694452047348,
+      "learning_rate": 6.285560182274008e-06,
+      "loss": 0.7744,
+      "step": 1130
+    },
+    {
+      "epoch": 0.43896804004620715,
+      "grad_norm": 0.23852677643299103,
+      "learning_rate": 6.225200208443936e-06,
+      "loss": 0.7985,
+      "step": 1140
+    },
+    {
+      "epoch": 0.44281863688871775,
+      "grad_norm": 0.21620960533618927,
+      "learning_rate": 6.16464966886616e-06,
+      "loss": 0.7838,
+      "step": 1150
+    },
+    {
+      "epoch": 0.44666923373122835,
+      "grad_norm": 0.274422824382782,
+      "learning_rate": 6.103917981478065e-06,
+      "loss": 0.8128,
+      "step": 1160
+    },
+    {
+      "epoch": 0.45051983057373896,
+      "grad_norm": 0.2193889319896698,
+      "learning_rate": 6.043014592392489e-06,
+      "loss": 0.8202,
+      "step": 1170
+    },
+    {
+      "epoch": 0.4543704274162495,
+      "grad_norm": 0.2095959484577179,
+      "learning_rate": 5.981948974428485e-06,
+      "loss": 0.7982,
+      "step": 1180
+    },
+    {
+      "epoch": 0.4582210242587601,
+      "grad_norm": 0.21365031599998474,
+      "learning_rate": 5.920730625637934e-06,
+      "loss": 0.7778,
+      "step": 1190
+    },
+    {
+      "epoch": 0.4620716211012707,
+      "grad_norm": 0.2463049441576004,
+      "learning_rate": 5.8593690678282355e-06,
+      "loss": 0.7837,
+      "step": 1200
+    },
+    {
+      "epoch": 0.4659222179437813,
+      "grad_norm": 0.2270236313343048,
+      "learning_rate": 5.797873845081301e-06,
+      "loss": 0.7823,
+      "step": 1210
+    },
+    {
+      "epoch": 0.46977281478629185,
+      "grad_norm": 0.22604891657829285,
+      "learning_rate": 5.736254522269075e-06,
+      "loss": 0.8083,
+      "step": 1220
+    },
+    {
+      "epoch": 0.47362341162880245,
+      "grad_norm": 0.20877705514431,
+      "learning_rate": 5.6745206835658386e-06,
+      "loss": 0.8022,
+      "step": 1230
+    },
+    {
+      "epoch": 0.47747400847131305,
+      "grad_norm": 0.21567608416080475,
+      "learning_rate": 5.612681930957495e-06,
+      "loss": 0.7811,
+      "step": 1240
+    },
+    {
+      "epoch": 0.48132460531382365,
+      "grad_norm": 0.23530715703964233,
+      "learning_rate": 5.550747882748095e-06,
+      "loss": 0.7759,
+      "step": 1250
+    },
+    {
+      "epoch": 0.48517520215633425,
+      "grad_norm": 0.26549071073532104,
+      "learning_rate": 5.4887281720638195e-06,
+      "loss": 0.7875,
+      "step": 1260
+    },
+    {
+      "epoch": 0.4890257989988448,
+      "grad_norm": 0.25496941804885864,
+      "learning_rate": 5.42663244535466e-06,
+      "loss": 0.7679,
+      "step": 1270
+    },
+    {
+      "epoch": 0.4928763958413554,
+      "grad_norm": 0.23020629584789276,
+      "learning_rate": 5.3644703608940205e-06,
+      "loss": 0.781,
+      "step": 1280
+    },
+    {
+      "epoch": 0.496726992683866,
+      "grad_norm": 0.2121354043483734,
+      "learning_rate": 5.302251587276491e-06,
+      "loss": 0.7945,
+      "step": 1290
+    },
+    {
+      "epoch": 0.5005775895263765,
+      "grad_norm": 0.2401570826768875,
+      "learning_rate": 5.2399858019140005e-06,
+      "loss": 0.784,
+      "step": 1300
+    },
+    {
+      "epoch": 0.5044281863688872,
+      "grad_norm": 0.23241759836673737,
+      "learning_rate": 5.177682689530615e-06,
+      "loss": 0.8001,
+      "step": 1310
+    },
+    {
+      "epoch": 0.5082787832113977,
+      "grad_norm": 0.22411543130874634,
+      "learning_rate": 5.11535194065619e-06,
+      "loss": 0.7745,
+      "step": 1320
+    },
+    {
+      "epoch": 0.5121293800539084,
+      "grad_norm": 0.23220741748809814,
+      "learning_rate": 5.053003250119113e-06,
+      "loss": 0.784,
+      "step": 1330
+    },
+    {
+      "epoch": 0.515979976896419,
+      "grad_norm": 0.21973451972007751,
+      "learning_rate": 4.9906463155383925e-06,
+      "loss": 0.7886,
+      "step": 1340
+    },
+    {
+      "epoch": 0.5198305737389295,
+      "grad_norm": 0.24693480134010315,
+      "learning_rate": 4.9282908358153046e-06,
+      "loss": 0.7838,
+      "step": 1350
+    },
+    {
+      "epoch": 0.5236811705814401,
+      "grad_norm": 0.23187513649463654,
+      "learning_rate": 4.8659465096248345e-06,
+      "loss": 0.802,
+      "step": 1360
+    },
+    {
+      "epoch": 0.5275317674239507,
+      "grad_norm": 0.2416222244501114,
+      "learning_rate": 4.803623033907169e-06,
+      "loss": 0.796,
+      "step": 1370
+    },
+    {
+      "epoch": 0.5313823642664613,
+      "grad_norm": 0.20501026511192322,
+      "learning_rate": 4.741330102359441e-06,
+      "loss": 0.807,
+      "step": 1380
+    },
+    {
+      "epoch": 0.5352329611089719,
+      "grad_norm": 0.22101648151874542,
+      "learning_rate": 4.679077403927996e-06,
+      "loss": 0.7838,
+      "step": 1390
+    },
+    {
+      "epoch": 0.5390835579514824,
+      "grad_norm": 0.2292109876871109,
+      "learning_rate": 4.616874621301373e-06,
+      "loss": 0.7838,
+      "step": 1400
+    },
+    {
+      "epoch": 0.5429341547939931,
+      "grad_norm": 0.2578872740268707,
+      "learning_rate": 4.554731429404293e-06,
+      "loss": 0.8004,
+      "step": 1410
+    },
+    {
+      "epoch": 0.5467847516365036,
+      "grad_norm": 0.2595241367816925,
+      "learning_rate": 4.492657493892822e-06,
+      "loss": 0.7873,
+      "step": 1420
+    },
+    {
+      "epoch": 0.5506353484790143,
+      "grad_norm": 0.23474010825157166,
+      "learning_rate": 4.430662469650999e-06,
+      "loss": 0.7991,
+      "step": 1430
+    },
+    {
+      "epoch": 0.5544859453215248,
+      "grad_norm": 0.21999529004096985,
+      "learning_rate": 4.368755999289118e-06,
+      "loss": 0.7969,
+      "step": 1440
+    },
+    {
+      "epoch": 0.5583365421640354,
+      "grad_norm": 0.22780506312847137,
+      "learning_rate": 4.306947711643949e-06,
+      "loss": 0.7831,
+      "step": 1450
+    },
+    {
+      "epoch": 0.562187139006546,
+      "grad_norm": 0.25624027848243713,
+      "learning_rate": 4.245247220281068e-06,
+      "loss": 0.7873,
+      "step": 1460
+    },
+    {
+      "epoch": 0.5660377358490566,
+      "grad_norm": 0.21565763652324677,
+      "learning_rate": 4.183664121999587e-06,
+      "loss": 0.7858,
+      "step": 1470
+    },
+    {
+      "epoch": 0.5698883326915672,
+      "grad_norm": 0.22319422662258148,
+      "learning_rate": 4.122207995339474e-06,
+      "loss": 0.7771,
+      "step": 1480
+    },
+    {
+      "epoch": 0.5737389295340778,
+      "grad_norm": 0.21457213163375854,
+      "learning_rate": 4.060888399091734e-06,
+      "loss": 0.7877,
+      "step": 1490
+    },
+    {
+      "epoch": 0.5775895263765883,
+      "grad_norm": 0.24044035375118256,
+      "learning_rate": 3.999714870811639e-06,
+      "loss": 0.7971,
+      "step": 1500
+    },
+    {
+      "epoch": 0.581440123219099,
+      "grad_norm": 0.21044811606407166,
+      "learning_rate": 3.938696925335275e-06,
+      "loss": 0.792,
+      "step": 1510
+    },
+    {
+      "epoch": 0.5852907200616095,
+      "grad_norm": 0.21338045597076416,
+      "learning_rate": 3.87784405329962e-06,
+      "loss": 0.7999,
+      "step": 1520
+    },
+    {
+      "epoch": 0.5891413169041202,
+      "grad_norm": 0.21751457452774048,
+      "learning_rate": 3.817165719666382e-06,
+      "loss": 0.801,
+      "step": 1530
+    },
+    {
+      "epoch": 0.5929919137466307,
+      "grad_norm": 0.23498059809207916,
+      "learning_rate": 3.756671362249841e-06,
+      "loss": 0.7999,
+      "step": 1540
+    },
+    {
+      "epoch": 0.5968425105891413,
+      "grad_norm": 0.22972846031188965,
+      "learning_rate": 3.6963703902488907e-06,
+      "loss": 0.7889,
+      "step": 1550
+    },
+    {
+      "epoch": 0.6006931074316519,
+      "grad_norm": 0.20918482542037964,
+      "learning_rate": 3.6362721827835655e-06,
+      "loss": 0.8045,
+      "step": 1560
+    },
+    {
+      "epoch": 0.6045437042741625,
+      "grad_norm": 0.22671908140182495,
+      "learning_rate": 3.576386087436212e-06,
+      "loss": 0.8053,
+      "step": 1570
+    },
+    {
+      "epoch": 0.608394301116673,
+      "grad_norm": 0.23626407980918884,
+      "learning_rate": 3.5167214187975913e-06,
+      "loss": 0.7992,
+      "step": 1580
+    },
+    {
+      "epoch": 0.6122448979591837,
+      "grad_norm": 0.21838879585266113,
+      "learning_rate": 3.457287457018086e-06,
+      "loss": 0.7934,
+      "step": 1590
+    },
+    {
+      "epoch": 0.6160954948016942,
+      "grad_norm": 0.22085261344909668,
+      "learning_rate": 3.3980934463643003e-06,
+      "loss": 0.7897,
+      "step": 1600
+    },
+    {
+      "epoch": 0.6199460916442049,
+      "grad_norm": 0.20952896773815155,
+      "learning_rate": 3.3391485937812053e-06,
+      "loss": 0.7677,
+      "step": 1610
+    },
+    {
+      "epoch": 0.6237966884867154,
+      "grad_norm": 0.2307884842157364,
+      "learning_rate": 3.280462067460115e-06,
+      "loss": 0.7692,
+      "step": 1620
+    },
+    {
+      "epoch": 0.627647285329226,
+      "grad_norm": 0.21258702874183655,
+      "learning_rate": 3.222042995412669e-06,
+      "loss": 0.7941,
+      "step": 1630
+    },
+    {
+      "epoch": 0.6314978821717366,
+      "grad_norm": 0.19305013120174408,
+      "learning_rate": 3.163900464051089e-06,
+      "loss": 0.785,
+      "step": 1640
+    },
+    {
+      "epoch": 0.6353484790142472,
+      "grad_norm": 0.23581798374652863,
+      "learning_rate": 3.1060435167748826e-06,
+      "loss": 0.7992,
+      "step": 1650
+    },
+    {
+      "epoch": 0.6391990758567578,
+      "grad_norm": 0.22209946811199188,
+      "learning_rate": 3.048481152564253e-06,
+      "loss": 0.7749,
+      "step": 1660
+    },
+    {
+      "epoch": 0.6430496726992684,
+      "grad_norm": 0.22124549746513367,
+      "learning_rate": 2.9912223245804017e-06,
+      "loss": 0.7931,
+      "step": 1670
+    },
+    {
+      "epoch": 0.6469002695417789,
+      "grad_norm": 0.23853899538516998,
+      "learning_rate": 2.934275938772981e-06,
+      "loss": 0.7871,
+      "step": 1680
+    },
+    {
+      "epoch": 0.6507508663842896,
+      "grad_norm": 0.2221257984638214,
+      "learning_rate": 2.8776508524948676e-06,
+      "loss": 0.7829,
+      "step": 1690
+    },
+    {
+      "epoch": 0.6546014632268001,
+      "grad_norm": 0.21256905794143677,
+      "learning_rate": 2.821355873124503e-06,
+      "loss": 0.786,
+      "step": 1700
+    },
+    {
+      "epoch": 0.6584520600693108,
+      "grad_norm": 0.23078301548957825,
+      "learning_rate": 2.7653997566960144e-06,
+      "loss": 0.7881,
+      "step": 1710
+    },
+    {
+      "epoch": 0.6623026569118213,
+      "grad_norm": 0.29431459307670593,
+      "learning_rate": 2.709791206537309e-06,
+      "loss": 0.7821,
+      "step": 1720
+    },
+    {
+      "epoch": 0.6661532537543319,
+      "grad_norm": 0.23275573551654816,
+      "learning_rate": 2.6545388719163747e-06,
+      "loss": 0.7909,
+      "step": 1730
+    },
+    {
+      "epoch": 0.6700038505968425,
+      "grad_norm": 0.22120822966098785,
+      "learning_rate": 2.599651346695979e-06,
+      "loss": 0.7983,
+      "step": 1740
+    },
+    {
+      "epoch": 0.6738544474393531,
+      "grad_norm": 0.21299047768115997,
+      "learning_rate": 2.545137167997007e-06,
+      "loss": 0.7894,
+      "step": 1750
+    },
+    {
+      "epoch": 0.6777050442818637,
+      "grad_norm": 0.220326229929924,
+      "learning_rate": 2.4910048148705972e-06,
+      "loss": 0.7861,
+      "step": 1760
+    },
+    {
+      "epoch": 0.6815556411243743,
+      "grad_norm": 0.230378195643425,
+      "learning_rate": 2.4372627069793327e-06,
+      "loss": 0.8183,
+      "step": 1770
+    },
+    {
+      "epoch": 0.6854062379668848,
+      "grad_norm": 0.1979997158050537,
+      "learning_rate": 2.3839192032876486e-06,
+      "loss": 0.7969,
+      "step": 1780
+    },
+    {
+      "epoch": 0.6892568348093955,
+      "grad_norm": 0.26156067848205566,
+      "learning_rate": 2.3309826007617047e-06,
+      "loss": 0.8118,
+      "step": 1790
+    },
+    {
+      "epoch": 0.693107431651906,
+      "grad_norm": 0.21007360517978668,
+      "learning_rate": 2.278461133078881e-06,
+      "loss": 0.7789,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6969580284944167,
+      "grad_norm": 0.23793993890285492,
+      "learning_rate": 2.2263629693471258e-06,
+      "loss": 0.774,
+      "step": 1810
+    },
+    {
+      "epoch": 0.7008086253369272,
+      "grad_norm": 0.20802199840545654,
+      "learning_rate": 2.1746962128343414e-06,
+      "loss": 0.7982,
+      "step": 1820
+    },
+    {
+      "epoch": 0.7046592221794378,
+      "grad_norm": 0.22921933233737946,
+      "learning_rate": 2.1234688997080183e-06,
+      "loss": 0.7797,
+      "step": 1830
+    },
+    {
+      "epoch": 0.7085098190219484,
+      "grad_norm": 0.21518725156784058,
+      "learning_rate": 2.0726889977853053e-06,
+      "loss": 0.784,
+      "step": 1840
+    },
+    {
+      "epoch": 0.712360415864459,
+      "grad_norm": 0.2265263944864273,
+      "learning_rate": 2.022364405293703e-06,
+      "loss": 0.7988,
+      "step": 1850
+    },
+    {
+      "epoch": 0.7162110127069696,
+      "grad_norm": 0.21258367598056793,
+      "learning_rate": 1.9725029496425814e-06,
+      "loss": 0.797,
+      "step": 1860
+    },
+    {
+      "epoch": 0.7200616095494802,
+      "grad_norm": 0.22828994691371918,
+      "learning_rate": 1.923112386205727e-06,
+      "loss": 0.7875,
+      "step": 1870
+    },
+    {
+      "epoch": 0.7239122063919907,
+      "grad_norm": 0.20674216747283936,
+      "learning_rate": 1.874200397115079e-06,
+      "loss": 0.7987,
+      "step": 1880
+    },
+    {
+      "epoch": 0.7277628032345014,
+      "grad_norm": 0.22343100607395172,
+      "learning_rate": 1.825774590065857e-06,
+      "loss": 0.7846,
+      "step": 1890
+    },
+    {
+      "epoch": 0.7316134000770119,
+      "grad_norm": 0.2263288050889969,
+      "learning_rate": 1.7778424971332813e-06,
+      "loss": 0.7953,
+      "step": 1900
+    },
+    {
+      "epoch": 0.7354639969195226,
+      "grad_norm": 0.20411835610866547,
+      "learning_rate": 1.7304115736010407e-06,
+      "loss": 0.7953,
+      "step": 1910
+    },
+    {
+      "epoch": 0.7393145937620331,
+      "grad_norm": 0.24508291482925415,
+      "learning_rate": 1.6834891968017108e-06,
+      "loss": 0.781,
+      "step": 1920
+    },
+    {
+      "epoch": 0.7431651906045437,
+      "grad_norm": 0.24442821741104126,
+      "learning_rate": 1.6370826649692884e-06,
+      "loss": 0.8039,
+      "step": 1930
+    },
+    {
+      "epoch": 0.7470157874470543,
+      "grad_norm": 0.21722131967544556,
+      "learning_rate": 1.5911991961040463e-06,
+      "loss": 0.7988,
+      "step": 1940
+    },
+    {
+      "epoch": 0.7508663842895649,
+      "grad_norm": 0.20581088960170746,
+      "learning_rate": 1.545845926849851e-06,
+      "loss": 0.7791,
+      "step": 1950
+    },
+    {
+      "epoch": 0.7547169811320755,
+      "grad_norm": 0.22773605585098267,
+      "learning_rate": 1.5010299113841397e-06,
+      "loss": 0.8084,
+      "step": 1960
+    },
+    {
+      "epoch": 0.7585675779745861,
+      "grad_norm": 0.22168083488941193,
+      "learning_rate": 1.4567581203207238e-06,
+      "loss": 0.7965,
+      "step": 1970
+    },
+    {
+      "epoch": 0.7624181748170966,
+      "grad_norm": 0.2288655936717987,
+      "learning_rate": 1.413037439625593e-06,
+      "loss": 0.7958,
+      "step": 1980
+    },
+    {
+      "epoch": 0.7662687716596073,
+      "grad_norm": 0.24186211824417114,
+      "learning_rate": 1.3698746695458847e-06,
+      "loss": 0.7828,
+      "step": 1990
+    },
+    {
+      "epoch": 0.7701193685021178,
+      "grad_norm": 0.21462500095367432,
+      "learning_rate": 1.3272765235521745e-06,
+      "loss": 0.7854,
+      "step": 2000
+    },
+    {
+      "epoch": 0.7739699653446284,
+      "grad_norm": 0.21529656648635864,
+      "learning_rate": 1.2852496272942877e-06,
+      "loss": 0.7906,
+      "step": 2010
+    },
+    {
+      "epoch": 0.777820562187139,
+      "grad_norm": 0.214616060256958,
+      "learning_rate": 1.2438005175707396e-06,
+      "loss": 0.7744,
+      "step": 2020
+    },
+    {
+      "epoch": 0.7816711590296496,
+      "grad_norm": 0.22429227828979492,
+      "learning_rate": 1.2029356413120297e-06,
+      "loss": 0.7702,
+      "step": 2030
+    },
+    {
+      "epoch": 0.7855217558721602,
+      "grad_norm": 0.22009506821632385,
+      "learning_rate": 1.1626613545778786e-06,
+      "loss": 0.7852,
+      "step": 2040
+    },
+    {
+      "epoch": 0.7893723527146708,
+      "grad_norm": 0.2121715247631073,
+      "learning_rate": 1.1229839215686294e-06,
+      "loss": 0.7873,
+      "step": 2050
+    },
+    {
+      "epoch": 0.7932229495571813,
+      "grad_norm": 0.20010986924171448,
+      "learning_rate": 1.0839095136509188e-06,
+      "loss": 0.7739,
+      "step": 2060
+    },
+    {
+      "epoch": 0.797073546399692,
+      "grad_norm": 0.2374793440103531,
+      "learning_rate": 1.045444208397791e-06,
+      "loss": 0.792,
+      "step": 2070
+    },
+    {
+      "epoch": 0.8009241432422025,
+      "grad_norm": 0.19836142659187317,
+      "learning_rate": 1.007593988643402e-06,
+      "loss": 0.7659,
+      "step": 2080
+    },
+    {
+      "epoch": 0.8047747400847132,
+      "grad_norm": 0.20639503002166748,
+      "learning_rate": 9.703647415524613e-07,
+      "loss": 0.7929,
+      "step": 2090
+    },
+    {
+      "epoch": 0.8086253369272237,
+      "grad_norm": 0.22045354545116425,
+      "learning_rate": 9.337622577045485e-07,
+      "loss": 0.7874,
+      "step": 2100
+    },
+    {
+      "epoch": 0.8124759337697343,
+      "grad_norm": 0.21670182049274445,
+      "learning_rate": 8.977922301934588e-07,
+      "loss": 0.7871,
+      "step": 2110
+    },
+    {
+      "epoch": 0.8163265306122449,
+      "grad_norm": 0.23518070578575134,
+      "learning_rate": 8.624602537417009e-07,
+      "loss": 0.8007,
+      "step": 2120
+    },
+    {
+      "epoch": 0.8201771274547555,
+      "grad_norm": 0.2300836145877838,
+      "learning_rate": 8.277718238303123e-07,
+      "loss": 0.7839,
+      "step": 2130
+    },
+    {
+      "epoch": 0.8240277242972661,
+      "grad_norm": 0.211968794465065,
+      "learning_rate": 7.937323358440935e-07,
+      "loss": 0.7853,
+      "step": 2140
+    },
+    {
+      "epoch": 0.8278783211397767,
+      "grad_norm": 0.2201559692621231,
+      "learning_rate": 7.603470842324223e-07,
+      "loss": 0.7759,
+      "step": 2150
+    },
+    {
+      "epoch": 0.8317289179822872,
+      "grad_norm": 0.21295641362667084,
+      "learning_rate": 7.276212616857575e-07,
+      "loss": 0.796,
+      "step": 2160
+    },
+    {
+      "epoch": 0.8355795148247979,
+      "grad_norm": 0.203803151845932,
+      "learning_rate": 6.955599583279837e-07,
+      "loss": 0.7912,
+      "step": 2170
+    },
+    {
+      "epoch": 0.8394301116673084,
+      "grad_norm": 0.19805125892162323,
+      "learning_rate": 6.641681609246981e-07,
+      "loss": 0.7837,
+      "step": 2180
+    },
+    {
+      "epoch": 0.8432807085098191,
+      "grad_norm": 0.23779939115047455,
+      "learning_rate": 6.334507521075711e-07,
+      "loss": 0.7881,
+      "step": 2190
+    },
+    {
+      "epoch": 0.8471313053523296,
+      "grad_norm": 0.22951640188694,
+      "learning_rate": 6.034125096149179e-07,
+      "loss": 0.7884,
+      "step": 2200
+    },
+    {
+      "epoch": 0.8509819021948402,
+      "grad_norm": 0.22755715250968933,
+      "learning_rate": 5.740581055485722e-07,
+      "loss": 0.7599,
+      "step": 2210
+    },
+    {
+      "epoch": 0.8548324990373508,
+      "grad_norm": 0.20284761488437653,
+      "learning_rate": 5.453921056471961e-07,
+      "loss": 0.7928,
+      "step": 2220
+    },
+    {
+      "epoch": 0.8586830958798614,
+      "grad_norm": 0.21459974348545074,
+      "learning_rate": 5.174189685761288e-07,
+      "loss": 0.786,
+      "step": 2230
+    },
+    {
+      "epoch": 0.862533692722372,
+      "grad_norm": 0.20141203701496124,
+      "learning_rate": 4.901430452338979e-07,
+      "loss": 0.7957,
+      "step": 2240
+    },
+    {
+      "epoch": 0.8663842895648826,
+      "grad_norm": 0.20365938544273376,
+      "learning_rate": 4.6356857807548286e-07,
+      "loss": 0.7852,
+      "step": 2250
+    },
+    {
+      "epoch": 0.8702348864073931,
+      "grad_norm": 0.2428484410047531,
+      "learning_rate": 4.376997004524541e-07,
+      "loss": 0.767,
+      "step": 2260
+    },
+    {
+      "epoch": 0.8740854832499038,
+      "grad_norm": 0.21171894669532776,
+      "learning_rate": 4.12540435970073e-07,
+      "loss": 0.7877,
+      "step": 2270
+    },
+    {
+      "epoch": 0.8779360800924143,
+      "grad_norm": 0.21615779399871826,
+      "learning_rate": 3.880946978614703e-07,
+      "loss": 0.7883,
+      "step": 2280
+    },
+    {
+      "epoch": 0.881786676934925,
+      "grad_norm": 0.20890730619430542,
+      "learning_rate": 3.643662883789878e-07,
+      "loss": 0.7982,
+      "step": 2290
+    },
+    {
+      "epoch": 0.8856372737774355,
+      "grad_norm": 0.18484315276145935,
+      "learning_rate": 3.41358898202781e-07,
+      "loss": 0.7764,
+      "step": 2300
+    },
+    {
+      "epoch": 0.889487870619946,
+      "grad_norm": 0.21299096941947937,
+      "learning_rate": 3.190761058667741e-07,
+      "loss": 0.7873,
+      "step": 2310
+    },
+    {
+      "epoch": 0.8933384674624567,
+      "grad_norm": 0.20092007517814636,
+      "learning_rate": 2.9752137720206463e-07,
+      "loss": 0.7916,
+      "step": 2320
+    },
+    {
+      "epoch": 0.8971890643049673,
+      "grad_norm": 0.22185243666172028,
+      "learning_rate": 2.7669806479785067e-07,
+      "loss": 0.8003,
+      "step": 2330
+    },
+    {
+      "epoch": 0.9010396611474779,
+      "grad_norm": 0.20740342140197754,
+      "learning_rate": 2.566094074799763e-07,
+      "loss": 0.7934,
+      "step": 2340
+    },
+    {
+      "epoch": 0.9048902579899885,
+      "grad_norm": 0.22336557507514954,
+      "learning_rate": 2.3725852980716557e-07,
+      "loss": 0.771,
+      "step": 2350
+    },
+    {
+      "epoch": 0.908740854832499,
+      "grad_norm": 0.19865953922271729,
+      "learning_rate": 2.1864844158503983e-07,
+      "loss": 0.7863,
+      "step": 2360
+    },
+    {
+      "epoch": 0.9125914516750097,
+      "grad_norm": 0.23999769985675812,
+      "learning_rate": 2.007820373979702e-07,
+      "loss": 0.7802,
+      "step": 2370
+    },
+    {
+      "epoch": 0.9164420485175202,
+      "grad_norm": 0.2057555466890335,
+      "learning_rate": 1.8366209615886532e-07,
+      "loss": 0.7891,
+      "step": 2380
+    },
+    {
+      "epoch": 0.9202926453600309,
+      "grad_norm": 0.19881361722946167,
+      "learning_rate": 1.6729128067693613e-07,
+      "loss": 0.8005,
+      "step": 2390
+    },
+    {
+      "epoch": 0.9241432422025414,
+      "grad_norm": 0.222417414188385,
+      "learning_rate": 1.5167213724353426e-07,
+      "loss": 0.795,
+      "step": 2400
+    },
+    {
+      "epoch": 0.927993839045052,
+      "grad_norm": 0.2092503160238266,
+      "learning_rate": 1.3680709523610058e-07,
+      "loss": 0.7731,
+      "step": 2410
+    },
+    {
+      "epoch": 0.9318444358875626,
+      "grad_norm": 0.20980864763259888,
+      "learning_rate": 1.2269846674030418e-07,
+      "loss": 0.8045,
+      "step": 2420
+    },
+    {
+      "epoch": 0.9356950327300732,
+      "grad_norm": 0.21088872849941254,
+      "learning_rate": 1.0934844619042595e-07,
+      "loss": 0.7967,
+      "step": 2430
+    },
+    {
+      "epoch": 0.9395456295725837,
+      "grad_norm": 0.2101893126964569,
+      "learning_rate": 9.675911002803961e-08,
+      "loss": 0.7889,
+      "step": 2440
+    },
+    {
+      "epoch": 0.9433962264150944,
+      "grad_norm": 0.2235390543937683,
+      "learning_rate": 8.493241637904381e-08,
+      "loss": 0.7766,
+      "step": 2450
+    },
+    {
+      "epoch": 0.9472468232576049,
+      "grad_norm": 0.23943668603897095,
+      "learning_rate": 7.387020474909689e-08,
+      "loss": 0.8034,
+      "step": 2460
+    },
+    {
+      "epoch": 0.9510974201001156,
+      "grad_norm": 0.2353707104921341,
+      "learning_rate": 6.357419573750856e-08,
+      "loss": 0.7988,
+      "step": 2470
+    },
+    {
+      "epoch": 0.9549480169426261,
+      "grad_norm": 0.21903729438781738,
+      "learning_rate": 5.4045990769613365e-08,
+      "loss": 0.7847,
+      "step": 2480
+    },
+    {
+      "epoch": 0.9587986137851366,
+      "grad_norm": 0.21340782940387726,
+      "learning_rate": 4.5287071847694434e-08,
+      "loss": 0.7855,
+      "step": 2490
+    },
+    {
+      "epoch": 0.9626492106276473,
+      "grad_norm": 0.19870266318321228,
+      "learning_rate": 3.7298801320467817e-08,
+      "loss": 0.7865,
+      "step": 2500
+    },
+    {
+      "epoch": 0.9664998074701578,
+      "grad_norm": 0.20546604692935944,
+      "learning_rate": 3.0082421671192576e-08,
+      "loss": 0.7851,
+      "step": 2510
+    },
+    {
+      "epoch": 0.9703504043126685,
+      "grad_norm": 0.21774867177009583,
+      "learning_rate": 2.3639055324410355e-08,
+      "loss": 0.7979,
+      "step": 2520
+    },
+    {
+      "epoch": 0.974201001155179,
+      "grad_norm": 0.21530194580554962,
+      "learning_rate": 1.7969704471369477e-08,
+      "loss": 0.7807,
+      "step": 2530
+    },
+    {
+      "epoch": 0.9780515979976896,
+      "grad_norm": 0.1971634179353714,
+      "learning_rate": 1.3075250914142413e-08,
+      "loss": 0.7795,
+      "step": 2540
+    },
+    {
+      "epoch": 0.9819021948402002,
+      "grad_norm": 0.20637072622776031,
+      "learning_rate": 8.956455928476049e-09,
+      "loss": 0.7966,
+      "step": 2550
+    },
+    {
+      "epoch": 0.9857527916827108,
+      "grad_norm": 0.21143928170204163,
+      "learning_rate": 5.6139601453786275e-09,
+      "loss": 0.7829,
+      "step": 2560
+    },
+    {
+      "epoch": 0.9896033885252215,
+      "grad_norm": 0.20963099598884583,
+      "learning_rate": 3.0482834514805602e-09,
+      "loss": 0.7768,
+      "step": 2570
+    },
+    {
+      "epoch": 0.993453985367732,
+      "grad_norm": 0.2084021270275116,
+      "learning_rate": 1.2598249081713366e-09,
+      "loss": 0.7688,
+      "step": 2580
+    },
+    {
+      "epoch": 0.9973045822102425,
+      "grad_norm": 0.2104540467262268,
+      "learning_rate": 2.488626895302826e-10,
+      "loss": 0.7989,
+      "step": 2590
+    },
+    {
+      "epoch": 1.0,
+      "step": 2597,
+      "total_flos": 5.740955908008051e+18,
+      "train_loss": 0.8011359955468543,
+      "train_runtime": 14676.4316,
+      "train_samples_per_second": 45.304,
+      "train_steps_per_second": 0.177
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2597,
+  "num_input_tokens_seen": 584317136,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.740955908008051e+18,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

continuous/smollm2/dino_vits16/finetune/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b175b46f981a19519b0e73772aaf84cd8d10307b1bdddbf4d0ff97dc067d5c6
+size 6264