Upload continuous evaluation checkpoints, batch 46

continuous/smollm2/webssl_dino300m_full2b_224/finetune/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a8036b19a2ac0926621a96db07c2752143122dfa8461d11b4908a124252fb5b
+size 4256661912

continuous/smollm2/webssl_dino300m_full2b_224/finetune/special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

continuous/smollm2/webssl_dino300m_full2b_224/finetune/tokenizer_config.json

@@ -0,0 +1,155 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

continuous/smollm2/webssl_dino300m_full2b_224/finetune/trainer_state.json

@@ -0,0 +1,1856 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 2597,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.003850596842510589,
+      "grad_norm": 2.849217176437378,
+      "learning_rate": 1.153846153846154e-06,
+      "loss": 1.2499,
+      "step": 10
+    },
+    {
+      "epoch": 0.007701193685021178,
+      "grad_norm": 2.105393171310425,
+      "learning_rate": 2.435897435897436e-06,
+      "loss": 1.1358,
+      "step": 20
+    },
+    {
+      "epoch": 0.011551790527531768,
+      "grad_norm": 2.063415050506592,
+      "learning_rate": 3.7179487179487184e-06,
+      "loss": 1.0335,
+      "step": 30
+    },
+    {
+      "epoch": 0.015402387370042356,
+      "grad_norm": 1.3648158311843872,
+      "learning_rate": 5e-06,
+      "loss": 0.9839,
+      "step": 40
+    },
+    {
+      "epoch": 0.019252984212552945,
+      "grad_norm": 1.318146824836731,
+      "learning_rate": 6.282051282051282e-06,
+      "loss": 0.9667,
+      "step": 50
+    },
+    {
+      "epoch": 0.023103581055063535,
+      "grad_norm": 0.29126107692718506,
+      "learning_rate": 7.564102564102564e-06,
+      "loss": 0.897,
+      "step": 60
+    },
+    {
+      "epoch": 0.026954177897574125,
+      "grad_norm": 0.23756088316440582,
+      "learning_rate": 8.846153846153847e-06,
+      "loss": 0.8513,
+      "step": 70
+    },
+    {
+      "epoch": 0.030804774740084712,
+      "grad_norm": 0.23362097144126892,
+      "learning_rate": 9.999996111488723e-06,
+      "loss": 0.8641,
+      "step": 80
+    },
+    {
+      "epoch": 0.0346553715825953,
+      "grad_norm": 0.2201012820005417,
+      "learning_rate": 9.999529497453782e-06,
+      "loss": 0.8517,
+      "step": 90
+    },
+    {
+      "epoch": 0.03850596842510589,
+      "grad_norm": 0.21237178146839142,
+      "learning_rate": 9.998285264324193e-06,
+      "loss": 0.8202,
+      "step": 100
+    },
+    {
+      "epoch": 0.04235656526761648,
+      "grad_norm": 0.2062198668718338,
+      "learning_rate": 9.996263605626052e-06,
+      "loss": 0.8105,
+      "step": 110
+    },
+    {
+      "epoch": 0.04620716211012707,
+      "grad_norm": 0.20854510366916656,
+      "learning_rate": 9.99346483580503e-06,
+      "loss": 0.8297,
+      "step": 120
+    },
+    {
+      "epoch": 0.05005775895263766,
+      "grad_norm": 0.19502753019332886,
+      "learning_rate": 9.989889390177467e-06,
+      "loss": 0.8077,
+      "step": 130
+    },
+    {
+      "epoch": 0.05390835579514825,
+      "grad_norm": 0.2161751389503479,
+      "learning_rate": 9.985537824862647e-06,
+      "loss": 0.8202,
+      "step": 140
+    },
+    {
+      "epoch": 0.05775895263765884,
+      "grad_norm": 0.21532166004180908,
+      "learning_rate": 9.980410816696317e-06,
+      "loss": 0.797,
+      "step": 150
+    },
+    {
+      "epoch": 0.061609549480169425,
+      "grad_norm": 0.20600679516792297,
+      "learning_rate": 9.97450916312541e-06,
+      "loss": 0.8022,
+      "step": 160
+    },
+    {
+      "epoch": 0.06546014632268002,
+      "grad_norm": 0.19071751832962036,
+      "learning_rate": 9.967833782084004e-06,
+      "loss": 0.8013,
+      "step": 170
+    },
+    {
+      "epoch": 0.0693107431651906,
+      "grad_norm": 0.2048383355140686,
+      "learning_rate": 9.960385711850554e-06,
+      "loss": 0.8347,
+      "step": 180
+    },
+    {
+      "epoch": 0.07316134000770119,
+      "grad_norm": 0.19958193600177765,
+      "learning_rate": 9.952166110886394e-06,
+      "loss": 0.821,
+      "step": 190
+    },
+    {
+      "epoch": 0.07701193685021178,
+      "grad_norm": 0.19884508848190308,
+      "learning_rate": 9.943176257655567e-06,
+      "loss": 0.8198,
+      "step": 200
+    },
+    {
+      "epoch": 0.08086253369272237,
+      "grad_norm": 0.20409642159938812,
+      "learning_rate": 9.933417550425952e-06,
+      "loss": 0.8054,
+      "step": 210
+    },
+    {
+      "epoch": 0.08471313053523297,
+      "grad_norm": 0.19160839915275574,
+      "learning_rate": 9.9228915070518e-06,
+      "loss": 0.7974,
+      "step": 220
+    },
+    {
+      "epoch": 0.08856372737774355,
+      "grad_norm": 0.1883227825164795,
+      "learning_rate": 9.911599764737632e-06,
+      "loss": 0.7922,
+      "step": 230
+    },
+    {
+      "epoch": 0.09241432422025414,
+      "grad_norm": 0.19248853623867035,
+      "learning_rate": 9.899544079783606e-06,
+      "loss": 0.7864,
+      "step": 240
+    },
+    {
+      "epoch": 0.09626492106276473,
+      "grad_norm": 0.19169457256793976,
+      "learning_rate": 9.886726327312338e-06,
+      "loss": 0.8029,
+      "step": 250
+    },
+    {
+      "epoch": 0.10011551790527531,
+      "grad_norm": 0.18853391706943512,
+      "learning_rate": 9.87314850097724e-06,
+      "loss": 0.8014,
+      "step": 260
+    },
+    {
+      "epoch": 0.1039661147477859,
+      "grad_norm": 0.1899716854095459,
+      "learning_rate": 9.858812712652452e-06,
+      "loss": 0.7832,
+      "step": 270
+    },
+    {
+      "epoch": 0.1078167115902965,
+      "grad_norm": 0.16914738714694977,
+      "learning_rate": 9.843721192104341e-06,
+      "loss": 0.7819,
+      "step": 280
+    },
+    {
+      "epoch": 0.11166730843280709,
+      "grad_norm": 0.18293850123882294,
+      "learning_rate": 9.827876286644698e-06,
+      "loss": 0.813,
+      "step": 290
+    },
+    {
+      "epoch": 0.11551790527531768,
+      "grad_norm": 0.1788962334394455,
+      "learning_rate": 9.811280460765643e-06,
+      "loss": 0.7701,
+      "step": 300
+    },
+    {
+      "epoch": 0.11936850211782826,
+      "grad_norm": 0.17531166970729828,
+      "learning_rate": 9.793936295756292e-06,
+      "loss": 0.7759,
+      "step": 310
+    },
+    {
+      "epoch": 0.12321909896033885,
+      "grad_norm": 0.19653576612472534,
+      "learning_rate": 9.775846489301279e-06,
+      "loss": 0.7839,
+      "step": 320
+    },
+    {
+      "epoch": 0.12706969580284944,
+      "grad_norm": 0.1804468333721161,
+      "learning_rate": 9.757013855061157e-06,
+      "loss": 0.7873,
+      "step": 330
+    },
+    {
+      "epoch": 0.13092029264536004,
+      "grad_norm": 0.19175902009010315,
+      "learning_rate": 9.737441322234758e-06,
+      "loss": 0.7897,
+      "step": 340
+    },
+    {
+      "epoch": 0.1347708894878706,
+      "grad_norm": 0.19713476300239563,
+      "learning_rate": 9.717131935103605e-06,
+      "loss": 0.795,
+      "step": 350
+    },
+    {
+      "epoch": 0.1386214863303812,
+      "grad_norm": 0.1849808543920517,
+      "learning_rate": 9.696088852558398e-06,
+      "loss": 0.7755,
+      "step": 360
+    },
+    {
+      "epoch": 0.1424720831728918,
+      "grad_norm": 0.1782224327325821,
+      "learning_rate": 9.674315347607686e-06,
+      "loss": 0.8032,
+      "step": 370
+    },
+    {
+      "epoch": 0.14632268001540238,
+      "grad_norm": 0.174220472574234,
+      "learning_rate": 9.651814806868796e-06,
+      "loss": 0.7678,
+      "step": 380
+    },
+    {
+      "epoch": 0.15017327685791299,
+      "grad_norm": 0.17979788780212402,
+      "learning_rate": 9.62859073004107e-06,
+      "loss": 0.772,
+      "step": 390
+    },
+    {
+      "epoch": 0.15402387370042356,
+      "grad_norm": 0.16711899638175964,
+      "learning_rate": 9.604646729361547e-06,
+      "loss": 0.7812,
+      "step": 400
+    },
+    {
+      "epoch": 0.15787447054293416,
+      "grad_norm": 0.1943427175283432,
+      "learning_rate": 9.579986529043099e-06,
+      "loss": 0.7777,
+      "step": 410
+    },
+    {
+      "epoch": 0.16172506738544473,
+      "grad_norm": 0.19390498101711273,
+      "learning_rate": 9.554613964695189e-06,
+      "loss": 0.7907,
+      "step": 420
+    },
+    {
+      "epoch": 0.16557566422795533,
+      "grad_norm": 0.1810373067855835,
+      "learning_rate": 9.528532982727281e-06,
+      "loss": 0.7704,
+      "step": 430
+    },
+    {
+      "epoch": 0.16942626107046593,
+      "grad_norm": 0.17690858244895935,
+      "learning_rate": 9.501747639735018e-06,
+      "loss": 0.7787,
+      "step": 440
+    },
+    {
+      "epoch": 0.1732768579129765,
+      "grad_norm": 0.18913953006267548,
+      "learning_rate": 9.474262101869272e-06,
+      "loss": 0.7638,
+      "step": 450
+    },
+    {
+      "epoch": 0.1771274547554871,
+      "grad_norm": 0.19607771933078766,
+      "learning_rate": 9.446080644188141e-06,
+      "loss": 0.7723,
+      "step": 460
+    },
+    {
+      "epoch": 0.18097805159799768,
+      "grad_norm": 0.171998992562294,
+      "learning_rate": 9.417207649992017e-06,
+      "loss": 0.7763,
+      "step": 470
+    },
+    {
+      "epoch": 0.18482864844050828,
+      "grad_norm": 0.19282227754592896,
+      "learning_rate": 9.387647610141816e-06,
+      "loss": 0.7757,
+      "step": 480
+    },
+    {
+      "epoch": 0.18867924528301888,
+      "grad_norm": 0.17418818175792694,
+      "learning_rate": 9.35740512236047e-06,
+      "loss": 0.7682,
+      "step": 490
+    },
+    {
+      "epoch": 0.19252984212552945,
+      "grad_norm": 0.1748940497636795,
+      "learning_rate": 9.326484890517809e-06,
+      "loss": 0.7709,
+      "step": 500
+    },
+    {
+      "epoch": 0.19638043896804006,
+      "grad_norm": 0.189378023147583,
+      "learning_rate": 9.294891723898926e-06,
+      "loss": 0.762,
+      "step": 510
+    },
+    {
+      "epoch": 0.20023103581055063,
+      "grad_norm": 0.18480119109153748,
+      "learning_rate": 9.262630536456153e-06,
+      "loss": 0.7854,
+      "step": 520
+    },
+    {
+      "epoch": 0.20408163265306123,
+      "grad_norm": 0.18480555713176727,
+      "learning_rate": 9.229706346044749e-06,
+      "loss": 0.7688,
+      "step": 530
+    },
+    {
+      "epoch": 0.2079322294955718,
+      "grad_norm": 0.18214379251003265,
+      "learning_rate": 9.196124273642422e-06,
+      "loss": 0.7784,
+      "step": 540
+    },
+    {
+      "epoch": 0.2117828263380824,
+      "grad_norm": 0.18211865425109863,
+      "learning_rate": 9.161889542552842e-06,
+      "loss": 0.7822,
+      "step": 550
+    },
+    {
+      "epoch": 0.215633423180593,
+      "grad_norm": 0.18195733428001404,
+      "learning_rate": 9.127007477593193e-06,
+      "loss": 0.7862,
+      "step": 560
+    },
+    {
+      "epoch": 0.21948402002310358,
+      "grad_norm": 0.1981562376022339,
+      "learning_rate": 9.091483504265974e-06,
+      "loss": 0.8023,
+      "step": 570
+    },
+    {
+      "epoch": 0.22333461686561418,
+      "grad_norm": 0.19192363321781158,
+      "learning_rate": 9.055323147915116e-06,
+      "loss": 0.7678,
+      "step": 580
+    },
+    {
+      "epoch": 0.22718521370812475,
+      "grad_norm": 0.17986682057380676,
+      "learning_rate": 9.018532032866585e-06,
+      "loss": 0.7855,
+      "step": 590
+    },
+    {
+      "epoch": 0.23103581055063535,
+      "grad_norm": 0.1640821397304535,
+      "learning_rate": 8.981115881553579e-06,
+      "loss": 0.7817,
+      "step": 600
+    },
+    {
+      "epoch": 0.23488640739314592,
+      "grad_norm": 0.19147734344005585,
+      "learning_rate": 8.943080513626476e-06,
+      "loss": 0.7991,
+      "step": 610
+    },
+    {
+      "epoch": 0.23873700423565652,
+      "grad_norm": 0.16887128353118896,
+      "learning_rate": 8.904431845047641e-06,
+      "loss": 0.7776,
+      "step": 620
+    },
+    {
+      "epoch": 0.24258760107816713,
+      "grad_norm": 0.1815192848443985,
+      "learning_rate": 8.86517588717128e-06,
+      "loss": 0.7893,
+      "step": 630
+    },
+    {
+      "epoch": 0.2464381979206777,
+      "grad_norm": 0.1807527244091034,
+      "learning_rate": 8.82531874580844e-06,
+      "loss": 0.7865,
+      "step": 640
+    },
+    {
+      "epoch": 0.25028879476318827,
+      "grad_norm": 0.16280747950077057,
+      "learning_rate": 8.784866620277314e-06,
+      "loss": 0.7466,
+      "step": 650
+    },
+    {
+      "epoch": 0.25413939160569887,
+      "grad_norm": 0.17407937347888947,
+      "learning_rate": 8.743825802439018e-06,
+      "loss": 0.775,
+      "step": 660
+    },
+    {
+      "epoch": 0.2579899884482095,
+      "grad_norm": 0.19154754281044006,
+      "learning_rate": 8.702202675718956e-06,
+      "loss": 0.7821,
+      "step": 670
+    },
+    {
+      "epoch": 0.2618405852907201,
+      "grad_norm": 0.16906993091106415,
+      "learning_rate": 8.660003714113959e-06,
+      "loss": 0.7652,
+      "step": 680
+    },
+    {
+      "epoch": 0.2656911821332307,
+      "grad_norm": 0.17373989522457123,
+      "learning_rate": 8.617235481185323e-06,
+      "loss": 0.7758,
+      "step": 690
+    },
+    {
+      "epoch": 0.2695417789757412,
+      "grad_norm": 0.18278847634792328,
+      "learning_rate": 8.573904629037931e-06,
+      "loss": 0.7866,
+      "step": 700
+    },
+    {
+      "epoch": 0.2733923758182518,
+      "grad_norm": 0.16509222984313965,
+      "learning_rate": 8.530017897285585e-06,
+      "loss": 0.7672,
+      "step": 710
+    },
+    {
+      "epoch": 0.2772429726607624,
+      "grad_norm": 0.17906999588012695,
+      "learning_rate": 8.485582112002746e-06,
+      "loss": 0.7673,
+      "step": 720
+    },
+    {
+      "epoch": 0.281093569503273,
+      "grad_norm": 0.17354834079742432,
+      "learning_rate": 8.440604184662811e-06,
+      "loss": 0.7793,
+      "step": 730
+    },
+    {
+      "epoch": 0.2849441663457836,
+      "grad_norm": 0.17759284377098083,
+      "learning_rate": 8.395091111063118e-06,
+      "loss": 0.7645,
+      "step": 740
+    },
+    {
+      "epoch": 0.28879476318829417,
+      "grad_norm": 0.16005605459213257,
+      "learning_rate": 8.349049970236822e-06,
+      "loss": 0.761,
+      "step": 750
+    },
+    {
+      "epoch": 0.29264536003080477,
+      "grad_norm": 0.17859211564064026,
+      "learning_rate": 8.302487923351842e-06,
+      "loss": 0.7584,
+      "step": 760
+    },
+    {
+      "epoch": 0.29649595687331537,
+      "grad_norm": 0.16163209080696106,
+      "learning_rate": 8.255412212597021e-06,
+      "loss": 0.7762,
+      "step": 770
+    },
+    {
+      "epoch": 0.30034655371582597,
+      "grad_norm": 0.18417637050151825,
+      "learning_rate": 8.207830160055688e-06,
+      "loss": 0.7706,
+      "step": 780
+    },
+    {
+      "epoch": 0.3041971505583365,
+      "grad_norm": 0.17162886261940002,
+      "learning_rate": 8.159749166566786e-06,
+      "loss": 0.7556,
+      "step": 790
+    },
+    {
+      "epoch": 0.3080477474008471,
+      "grad_norm": 0.18247470259666443,
+      "learning_rate": 8.11117671057377e-06,
+      "loss": 0.7772,
+      "step": 800
+    },
+    {
+      "epoch": 0.3118983442433577,
+      "grad_norm": 0.1651126891374588,
+      "learning_rate": 8.062120346961404e-06,
+      "loss": 0.776,
+      "step": 810
+    },
+    {
+      "epoch": 0.3157489410858683,
+      "grad_norm": 0.17240431904792786,
+      "learning_rate": 8.012587705880705e-06,
+      "loss": 0.7737,
+      "step": 820
+    },
+    {
+      "epoch": 0.3195995379283789,
+      "grad_norm": 0.17988134920597076,
+      "learning_rate": 7.962586491562146e-06,
+      "loss": 0.7594,
+      "step": 830
+    },
+    {
+      "epoch": 0.32345013477088946,
+      "grad_norm": 0.18402262032032013,
+      "learning_rate": 7.912124481117357e-06,
+      "loss": 0.7593,
+      "step": 840
+    },
+    {
+      "epoch": 0.32730073161340006,
+      "grad_norm": 0.1640693098306656,
+      "learning_rate": 7.861209523329494e-06,
+      "loss": 0.7677,
+      "step": 850
+    },
+    {
+      "epoch": 0.33115132845591067,
+      "grad_norm": 0.17969422042369843,
+      "learning_rate": 7.809849537432432e-06,
+      "loss": 0.7615,
+      "step": 860
+    },
+    {
+      "epoch": 0.33500192529842127,
+      "grad_norm": 0.16457566618919373,
+      "learning_rate": 7.758052511879035e-06,
+      "loss": 0.764,
+      "step": 870
+    },
+    {
+      "epoch": 0.33885252214093187,
+      "grad_norm": 0.1726258546113968,
+      "learning_rate": 7.705826503098642e-06,
+      "loss": 0.7671,
+      "step": 880
+    },
+    {
+      "epoch": 0.3427031189834424,
+      "grad_norm": 0.17757385969161987,
+      "learning_rate": 7.653179634243979e-06,
+      "loss": 0.7598,
+      "step": 890
+    },
+    {
+      "epoch": 0.346553715825953,
+      "grad_norm": 0.16488133370876312,
+      "learning_rate": 7.600120093927696e-06,
+      "loss": 0.7637,
+      "step": 900
+    },
+    {
+      "epoch": 0.3504043126684636,
+      "grad_norm": 0.18766054511070251,
+      "learning_rate": 7.546656134948729e-06,
+      "loss": 0.7548,
+      "step": 910
+    },
+    {
+      "epoch": 0.3542549095109742,
+      "grad_norm": 0.1680602878332138,
+      "learning_rate": 7.492796073008661e-06,
+      "loss": 0.7597,
+      "step": 920
+    },
+    {
+      "epoch": 0.3581055063534848,
+      "grad_norm": 0.19374313950538635,
+      "learning_rate": 7.438548285418326e-06,
+      "loss": 0.7412,
+      "step": 930
+    },
+    {
+      "epoch": 0.36195610319599536,
+      "grad_norm": 0.18212372064590454,
+      "learning_rate": 7.3839212097948e-06,
+      "loss": 0.7725,
+      "step": 940
+    },
+    {
+      "epoch": 0.36580670003850596,
+      "grad_norm": 0.18580561876296997,
+      "learning_rate": 7.328923342749039e-06,
+      "loss": 0.7877,
+      "step": 950
+    },
+    {
+      "epoch": 0.36965729688101656,
+      "grad_norm": 0.17535936832427979,
+      "learning_rate": 7.273563238564319e-06,
+      "loss": 0.783,
+      "step": 960
+    },
+    {
+      "epoch": 0.37350789372352716,
+      "grad_norm": 0.16924476623535156,
+      "learning_rate": 7.217849507865724e-06,
+      "loss": 0.754,
+      "step": 970
+    },
+    {
+      "epoch": 0.37735849056603776,
+      "grad_norm": 0.16980408132076263,
+      "learning_rate": 7.161790816280858e-06,
+      "loss": 0.7616,
+      "step": 980
+    },
+    {
+      "epoch": 0.3812090874085483,
+      "grad_norm": 0.17926250398159027,
+      "learning_rate": 7.1053958830920125e-06,
+      "loss": 0.7518,
+      "step": 990
+    },
+    {
+      "epoch": 0.3850596842510589,
+      "grad_norm": 0.1729089617729187,
+      "learning_rate": 7.0486734798799715e-06,
+      "loss": 0.7604,
+      "step": 1000
+    },
+    {
+      "epoch": 0.3889102810935695,
+      "grad_norm": 0.17128226161003113,
+      "learning_rate": 6.991632429159708e-06,
+      "loss": 0.7804,
+      "step": 1010
+    },
+    {
+      "epoch": 0.3927608779360801,
+      "grad_norm": 0.19183418154716492,
+      "learning_rate": 6.934281603008132e-06,
+      "loss": 0.7787,
+      "step": 1020
+    },
+    {
+      "epoch": 0.39661147477859066,
+      "grad_norm": 0.15790912508964539,
+      "learning_rate": 6.876629921684143e-06,
+      "loss": 0.7751,
+      "step": 1030
+    },
+    {
+      "epoch": 0.40046207162110126,
+      "grad_norm": 0.1784982532262802,
+      "learning_rate": 6.818686352241199e-06,
+      "loss": 0.7638,
+      "step": 1040
+    },
+    {
+      "epoch": 0.40431266846361186,
+      "grad_norm": 0.15714190900325775,
+      "learning_rate": 6.760459907132573e-06,
+      "loss": 0.7601,
+      "step": 1050
+    },
+    {
+      "epoch": 0.40816326530612246,
+      "grad_norm": 0.1586635559797287,
+      "learning_rate": 6.701959642809589e-06,
+      "loss": 0.7655,
+      "step": 1060
+    },
+    {
+      "epoch": 0.41201386214863306,
+      "grad_norm": 0.1776939183473587,
+      "learning_rate": 6.643194658312987e-06,
+      "loss": 0.7578,
+      "step": 1070
+    },
+    {
+      "epoch": 0.4158644589911436,
+      "grad_norm": 0.1723143607378006,
+      "learning_rate": 6.584174093857676e-06,
+      "loss": 0.7827,
+      "step": 1080
+    },
+    {
+      "epoch": 0.4197150558336542,
+      "grad_norm": 0.18426714837551117,
+      "learning_rate": 6.5249071294110735e-06,
+      "loss": 0.7648,
+      "step": 1090
+    },
+    {
+      "epoch": 0.4235656526761648,
+      "grad_norm": 0.1797376573085785,
+      "learning_rate": 6.465402983265274e-06,
+      "loss": 0.777,
+      "step": 1100
+    },
+    {
+      "epoch": 0.4274162495186754,
+      "grad_norm": 0.18367977440357208,
+      "learning_rate": 6.405670910603243e-06,
+      "loss": 0.7702,
+      "step": 1110
+    },
+    {
+      "epoch": 0.431266846361186,
+      "grad_norm": 0.1841287463903427,
+      "learning_rate": 6.345720202059291e-06,
+      "loss": 0.7678,
+      "step": 1120
+    },
+    {
+      "epoch": 0.43511744320369655,
+      "grad_norm": 0.1768961250782013,
+      "learning_rate": 6.285560182274008e-06,
+      "loss": 0.7483,
+      "step": 1130
+    },
+    {
+      "epoch": 0.43896804004620715,
+      "grad_norm": 0.18733657896518707,
+      "learning_rate": 6.225200208443936e-06,
+      "loss": 0.7725,
+      "step": 1140
+    },
+    {
+      "epoch": 0.44281863688871775,
+      "grad_norm": 0.17633751034736633,
+      "learning_rate": 6.16464966886616e-06,
+      "loss": 0.7572,
+      "step": 1150
+    },
+    {
+      "epoch": 0.44666923373122835,
+      "grad_norm": 0.20781801640987396,
+      "learning_rate": 6.103917981478065e-06,
+      "loss": 0.7852,
+      "step": 1160
+    },
+    {
+      "epoch": 0.45051983057373896,
+      "grad_norm": 0.16702428460121155,
+      "learning_rate": 6.043014592392489e-06,
+      "loss": 0.7931,
+      "step": 1170
+    },
+    {
+      "epoch": 0.4543704274162495,
+      "grad_norm": 0.16834202408790588,
+      "learning_rate": 5.981948974428485e-06,
+      "loss": 0.7726,
+      "step": 1180
+    },
+    {
+      "epoch": 0.4582210242587601,
+      "grad_norm": 0.16285577416419983,
+      "learning_rate": 5.920730625637934e-06,
+      "loss": 0.7515,
+      "step": 1190
+    },
+    {
+      "epoch": 0.4620716211012707,
+      "grad_norm": 0.17477785050868988,
+      "learning_rate": 5.8593690678282355e-06,
+      "loss": 0.7582,
+      "step": 1200
+    },
+    {
+      "epoch": 0.4659222179437813,
+      "grad_norm": 0.17220604419708252,
+      "learning_rate": 5.797873845081301e-06,
+      "loss": 0.756,
+      "step": 1210
+    },
+    {
+      "epoch": 0.46977281478629185,
+      "grad_norm": 0.17277970910072327,
+      "learning_rate": 5.736254522269075e-06,
+      "loss": 0.7827,
+      "step": 1220
+    },
+    {
+      "epoch": 0.47362341162880245,
+      "grad_norm": 0.16617555916309357,
+      "learning_rate": 5.6745206835658386e-06,
+      "loss": 0.7762,
+      "step": 1230
+    },
+    {
+      "epoch": 0.47747400847131305,
+      "grad_norm": 0.1658850610256195,
+      "learning_rate": 5.612681930957495e-06,
+      "loss": 0.7556,
+      "step": 1240
+    },
+    {
+      "epoch": 0.48132460531382365,
+      "grad_norm": 0.17557743191719055,
+      "learning_rate": 5.550747882748095e-06,
+      "loss": 0.7491,
+      "step": 1250
+    },
+    {
+      "epoch": 0.48517520215633425,
+      "grad_norm": 0.18733975291252136,
+      "learning_rate": 5.4887281720638195e-06,
+      "loss": 0.7607,
+      "step": 1260
+    },
+    {
+      "epoch": 0.4890257989988448,
+      "grad_norm": 0.18563212454319,
+      "learning_rate": 5.42663244535466e-06,
+      "loss": 0.7405,
+      "step": 1270
+    },
+    {
+      "epoch": 0.4928763958413554,
+      "grad_norm": 0.179581880569458,
+      "learning_rate": 5.3644703608940205e-06,
+      "loss": 0.7557,
+      "step": 1280
+    },
+    {
+      "epoch": 0.496726992683866,
+      "grad_norm": 0.16690105199813843,
+      "learning_rate": 5.302251587276491e-06,
+      "loss": 0.7675,
+      "step": 1290
+    },
+    {
+      "epoch": 0.5005775895263765,
+      "grad_norm": 0.17263855040073395,
+      "learning_rate": 5.2399858019140005e-06,
+      "loss": 0.758,
+      "step": 1300
+    },
+    {
+      "epoch": 0.5044281863688872,
+      "grad_norm": 0.17454616725444794,
+      "learning_rate": 5.177682689530615e-06,
+      "loss": 0.7761,
+      "step": 1310
+    },
+    {
+      "epoch": 0.5082787832113977,
+      "grad_norm": 0.18240705132484436,
+      "learning_rate": 5.11535194065619e-06,
+      "loss": 0.7485,
+      "step": 1320
+    },
+    {
+      "epoch": 0.5121293800539084,
+      "grad_norm": 0.18256030976772308,
+      "learning_rate": 5.053003250119113e-06,
+      "loss": 0.758,
+      "step": 1330
+    },
+    {
+      "epoch": 0.515979976896419,
+      "grad_norm": 0.16959261894226074,
+      "learning_rate": 4.9906463155383925e-06,
+      "loss": 0.7612,
+      "step": 1340
+    },
+    {
+      "epoch": 0.5198305737389295,
+      "grad_norm": 0.17303021252155304,
+      "learning_rate": 4.9282908358153046e-06,
+      "loss": 0.7572,
+      "step": 1350
+    },
+    {
+      "epoch": 0.5236811705814401,
+      "grad_norm": 0.18690839409828186,
+      "learning_rate": 4.8659465096248345e-06,
+      "loss": 0.7759,
+      "step": 1360
+    },
+    {
+      "epoch": 0.5275317674239507,
+      "grad_norm": 0.1792057454586029,
+      "learning_rate": 4.803623033907169e-06,
+      "loss": 0.7694,
+      "step": 1370
+    },
+    {
+      "epoch": 0.5313823642664613,
+      "grad_norm": 0.164366215467453,
+      "learning_rate": 4.741330102359441e-06,
+      "loss": 0.7811,
+      "step": 1380
+    },
+    {
+      "epoch": 0.5352329611089719,
+      "grad_norm": 0.17595066130161285,
+      "learning_rate": 4.679077403927996e-06,
+      "loss": 0.7577,
+      "step": 1390
+    },
+    {
+      "epoch": 0.5390835579514824,
+      "grad_norm": 0.17264395952224731,
+      "learning_rate": 4.616874621301373e-06,
+      "loss": 0.7573,
+      "step": 1400
+    },
+    {
+      "epoch": 0.5429341547939931,
+      "grad_norm": 0.181438609957695,
+      "learning_rate": 4.554731429404293e-06,
+      "loss": 0.7739,
+      "step": 1410
+    },
+    {
+      "epoch": 0.5467847516365036,
+      "grad_norm": 0.18779264390468597,
+      "learning_rate": 4.492657493892822e-06,
+      "loss": 0.7602,
+      "step": 1420
+    },
+    {
+      "epoch": 0.5506353484790143,
+      "grad_norm": 0.17987701296806335,
+      "learning_rate": 4.430662469650999e-06,
+      "loss": 0.7723,
+      "step": 1430
+    },
+    {
+      "epoch": 0.5544859453215248,
+      "grad_norm": 0.18287141621112823,
+      "learning_rate": 4.368755999289118e-06,
+      "loss": 0.7723,
+      "step": 1440
+    },
+    {
+      "epoch": 0.5583365421640354,
+      "grad_norm": 0.18473967909812927,
+      "learning_rate": 4.306947711643949e-06,
+      "loss": 0.7577,
+      "step": 1450
+    },
+    {
+      "epoch": 0.562187139006546,
+      "grad_norm": 0.1783231943845749,
+      "learning_rate": 4.245247220281068e-06,
+      "loss": 0.7619,
+      "step": 1460
+    },
+    {
+      "epoch": 0.5660377358490566,
+      "grad_norm": 0.17187581956386566,
+      "learning_rate": 4.183664121999587e-06,
+      "loss": 0.7584,
+      "step": 1470
+    },
+    {
+      "epoch": 0.5698883326915672,
+      "grad_norm": 0.18063901364803314,
+      "learning_rate": 4.122207995339474e-06,
+      "loss": 0.7514,
+      "step": 1480
+    },
+    {
+      "epoch": 0.5737389295340778,
+      "grad_norm": 0.17112894356250763,
+      "learning_rate": 4.060888399091734e-06,
+      "loss": 0.7601,
+      "step": 1490
+    },
+    {
+      "epoch": 0.5775895263765883,
+      "grad_norm": 0.16773101687431335,
+      "learning_rate": 3.999714870811639e-06,
+      "loss": 0.7698,
+      "step": 1500
+    },
+    {
+      "epoch": 0.581440123219099,
+      "grad_norm": 0.17507511377334595,
+      "learning_rate": 3.938696925335275e-06,
+      "loss": 0.7645,
+      "step": 1510
+    },
+    {
+      "epoch": 0.5852907200616095,
+      "grad_norm": 0.17689302563667297,
+      "learning_rate": 3.87784405329962e-06,
+      "loss": 0.7731,
+      "step": 1520
+    },
+    {
+      "epoch": 0.5891413169041202,
+      "grad_norm": 0.1604737490415573,
+      "learning_rate": 3.817165719666382e-06,
+      "loss": 0.774,
+      "step": 1530
+    },
+    {
+      "epoch": 0.5929919137466307,
+      "grad_norm": 0.17282108962535858,
+      "learning_rate": 3.756671362249841e-06,
+      "loss": 0.7726,
+      "step": 1540
+    },
+    {
+      "epoch": 0.5968425105891413,
+      "grad_norm": 0.1620083898305893,
+      "learning_rate": 3.6963703902488907e-06,
+      "loss": 0.762,
+      "step": 1550
+    },
+    {
+      "epoch": 0.6006931074316519,
+      "grad_norm": 0.16276811063289642,
+      "learning_rate": 3.6362721827835655e-06,
+      "loss": 0.7764,
+      "step": 1560
+    },
+    {
+      "epoch": 0.6045437042741625,
+      "grad_norm": 0.1816469132900238,
+      "learning_rate": 3.576386087436212e-06,
+      "loss": 0.7787,
+      "step": 1570
+    },
+    {
+      "epoch": 0.608394301116673,
+      "grad_norm": 0.18503008782863617,
+      "learning_rate": 3.5167214187975913e-06,
+      "loss": 0.7715,
+      "step": 1580
+    },
+    {
+      "epoch": 0.6122448979591837,
+      "grad_norm": 0.15654872357845306,
+      "learning_rate": 3.457287457018086e-06,
+      "loss": 0.7665,
+      "step": 1590
+    },
+    {
+      "epoch": 0.6160954948016942,
+      "grad_norm": 0.16830459237098694,
+      "learning_rate": 3.3980934463643003e-06,
+      "loss": 0.7618,
+      "step": 1600
+    },
+    {
+      "epoch": 0.6199460916442049,
+      "grad_norm": 0.1632918417453766,
+      "learning_rate": 3.3391485937812053e-06,
+      "loss": 0.7405,
+      "step": 1610
+    },
+    {
+      "epoch": 0.6237966884867154,
+      "grad_norm": 0.1781202107667923,
+      "learning_rate": 3.280462067460115e-06,
+      "loss": 0.7416,
+      "step": 1620
+    },
+    {
+      "epoch": 0.627647285329226,
+      "grad_norm": 0.17197808623313904,
+      "learning_rate": 3.222042995412669e-06,
+      "loss": 0.7667,
+      "step": 1630
+    },
+    {
+      "epoch": 0.6314978821717366,
+      "grad_norm": 0.16060614585876465,
+      "learning_rate": 3.163900464051089e-06,
+      "loss": 0.7584,
+      "step": 1640
+    },
+    {
+      "epoch": 0.6353484790142472,
+      "grad_norm": 0.17147503793239594,
+      "learning_rate": 3.1060435167748826e-06,
+      "loss": 0.7722,
+      "step": 1650
+    },
+    {
+      "epoch": 0.6391990758567578,
+      "grad_norm": 0.1709176003932953,
+      "learning_rate": 3.048481152564253e-06,
+      "loss": 0.7486,
+      "step": 1660
+    },
+    {
+      "epoch": 0.6430496726992684,
+      "grad_norm": 0.18063513934612274,
+      "learning_rate": 2.9912223245804017e-06,
+      "loss": 0.766,
+      "step": 1670
+    },
+    {
+      "epoch": 0.6469002695417789,
+      "grad_norm": 0.17484848201274872,
+      "learning_rate": 2.934275938772981e-06,
+      "loss": 0.7612,
+      "step": 1680
+    },
+    {
+      "epoch": 0.6507508663842896,
+      "grad_norm": 0.18083792924880981,
+      "learning_rate": 2.8776508524948676e-06,
+      "loss": 0.7538,
+      "step": 1690
+    },
+    {
+      "epoch": 0.6546014632268001,
+      "grad_norm": 0.1660001575946808,
+      "learning_rate": 2.821355873124503e-06,
+      "loss": 0.7581,
+      "step": 1700
+    },
+    {
+      "epoch": 0.6584520600693108,
+      "grad_norm": 0.1767004281282425,
+      "learning_rate": 2.7653997566960144e-06,
+      "loss": 0.7627,
+      "step": 1710
+    },
+    {
+      "epoch": 0.6623026569118213,
+      "grad_norm": 0.19084595143795013,
+      "learning_rate": 2.709791206537309e-06,
+      "loss": 0.7559,
+      "step": 1720
+    },
+    {
+      "epoch": 0.6661532537543319,
+      "grad_norm": 0.18900954723358154,
+      "learning_rate": 2.6545388719163747e-06,
+      "loss": 0.7643,
+      "step": 1730
+    },
+    {
+      "epoch": 0.6700038505968425,
+      "grad_norm": 0.17421260476112366,
+      "learning_rate": 2.599651346695979e-06,
+      "loss": 0.7702,
+      "step": 1740
+    },
+    {
+      "epoch": 0.6738544474393531,
+      "grad_norm": 0.1659581959247589,
+      "learning_rate": 2.545137167997007e-06,
+      "loss": 0.763,
+      "step": 1750
+    },
+    {
+      "epoch": 0.6777050442818637,
+      "grad_norm": 0.1668090969324112,
+      "learning_rate": 2.4910048148705972e-06,
+      "loss": 0.7568,
+      "step": 1760
+    },
+    {
+      "epoch": 0.6815556411243743,
+      "grad_norm": 0.1828475296497345,
+      "learning_rate": 2.4372627069793327e-06,
+      "loss": 0.7911,
+      "step": 1770
+    },
+    {
+      "epoch": 0.6854062379668848,
+      "grad_norm": 0.1683444231748581,
+      "learning_rate": 2.3839192032876486e-06,
+      "loss": 0.769,
+      "step": 1780
+    },
+    {
+      "epoch": 0.6892568348093955,
+      "grad_norm": 0.184872105717659,
+      "learning_rate": 2.3309826007617047e-06,
+      "loss": 0.7849,
+      "step": 1790
+    },
+    {
+      "epoch": 0.693107431651906,
+      "grad_norm": 0.1714065819978714,
+      "learning_rate": 2.278461133078881e-06,
+      "loss": 0.7524,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6969580284944167,
+      "grad_norm": 0.17445909976959229,
+      "learning_rate": 2.2263629693471258e-06,
+      "loss": 0.7467,
+      "step": 1810
+    },
+    {
+      "epoch": 0.7008086253369272,
+      "grad_norm": 0.16486051678657532,
+      "learning_rate": 2.1746962128343414e-06,
+      "loss": 0.7703,
+      "step": 1820
+    },
+    {
+      "epoch": 0.7046592221794378,
+      "grad_norm": 0.17691953480243683,
+      "learning_rate": 2.1234688997080183e-06,
+      "loss": 0.7526,
+      "step": 1830
+    },
+    {
+      "epoch": 0.7085098190219484,
+      "grad_norm": 0.16957232356071472,
+      "learning_rate": 2.0726889977853053e-06,
+      "loss": 0.7575,
+      "step": 1840
+    },
+    {
+      "epoch": 0.712360415864459,
+      "grad_norm": 0.1765242964029312,
+      "learning_rate": 2.022364405293703e-06,
+      "loss": 0.7722,
+      "step": 1850
+    },
+    {
+      "epoch": 0.7162110127069696,
+      "grad_norm": 0.16795112192630768,
+      "learning_rate": 1.9725029496425814e-06,
+      "loss": 0.77,
+      "step": 1860
+    },
+    {
+      "epoch": 0.7200616095494802,
+      "grad_norm": 0.17752595245838165,
+      "learning_rate": 1.923112386205727e-06,
+      "loss": 0.7602,
+      "step": 1870
+    },
+    {
+      "epoch": 0.7239122063919907,
+      "grad_norm": 0.1708524078130722,
+      "learning_rate": 1.874200397115079e-06,
+      "loss": 0.7713,
+      "step": 1880
+    },
+    {
+      "epoch": 0.7277628032345014,
+      "grad_norm": 0.1738175004720688,
+      "learning_rate": 1.825774590065857e-06,
+      "loss": 0.7566,
+      "step": 1890
+    },
+    {
+      "epoch": 0.7316134000770119,
+      "grad_norm": 0.18926113843917847,
+      "learning_rate": 1.7778424971332813e-06,
+      "loss": 0.7672,
+      "step": 1900
+    },
+    {
+      "epoch": 0.7354639969195226,
+      "grad_norm": 0.1627325564622879,
+      "learning_rate": 1.7304115736010407e-06,
+      "loss": 0.7668,
+      "step": 1910
+    },
+    {
+      "epoch": 0.7393145937620331,
+      "grad_norm": 0.16836793720722198,
+      "learning_rate": 1.6834891968017108e-06,
+      "loss": 0.7535,
+      "step": 1920
+    },
+    {
+      "epoch": 0.7431651906045437,
+      "grad_norm": 0.1788535714149475,
+      "learning_rate": 1.6370826649692884e-06,
+      "loss": 0.775,
+      "step": 1930
+    },
+    {
+      "epoch": 0.7470157874470543,
+      "grad_norm": 0.16693656146526337,
+      "learning_rate": 1.5911991961040463e-06,
+      "loss": 0.7718,
+      "step": 1940
+    },
+    {
+      "epoch": 0.7508663842895649,
+      "grad_norm": 0.17377734184265137,
+      "learning_rate": 1.545845926849851e-06,
+      "loss": 0.7534,
+      "step": 1950
+    },
+    {
+      "epoch": 0.7547169811320755,
+      "grad_norm": 0.18540462851524353,
+      "learning_rate": 1.5010299113841397e-06,
+      "loss": 0.7805,
+      "step": 1960
+    },
+    {
+      "epoch": 0.7585675779745861,
+      "grad_norm": 0.17670921981334686,
+      "learning_rate": 1.4567581203207238e-06,
+      "loss": 0.769,
+      "step": 1970
+    },
+    {
+      "epoch": 0.7624181748170966,
+      "grad_norm": 0.18759745359420776,
+      "learning_rate": 1.413037439625593e-06,
+      "loss": 0.7689,
+      "step": 1980
+    },
+    {
+      "epoch": 0.7662687716596073,
+      "grad_norm": 0.19500567018985748,
+      "learning_rate": 1.3698746695458847e-06,
+      "loss": 0.7566,
+      "step": 1990
+    },
+    {
+      "epoch": 0.7701193685021178,
+      "grad_norm": 0.16680112481117249,
+      "learning_rate": 1.3272765235521745e-06,
+      "loss": 0.758,
+      "step": 2000
+    },
+    {
+      "epoch": 0.7739699653446284,
+      "grad_norm": 0.17124028503894806,
+      "learning_rate": 1.2852496272942877e-06,
+      "loss": 0.7637,
+      "step": 2010
+    },
+    {
+      "epoch": 0.777820562187139,
+      "grad_norm": 0.16327917575836182,
+      "learning_rate": 1.2438005175707396e-06,
+      "loss": 0.7468,
+      "step": 2020
+    },
+    {
+      "epoch": 0.7816711590296496,
+      "grad_norm": 0.17247430980205536,
+      "learning_rate": 1.2029356413120297e-06,
+      "loss": 0.7434,
+      "step": 2030
+    },
+    {
+      "epoch": 0.7855217558721602,
+      "grad_norm": 0.1718081831932068,
+      "learning_rate": 1.1626613545778786e-06,
+      "loss": 0.7577,
+      "step": 2040
+    },
+    {
+      "epoch": 0.7893723527146708,
+      "grad_norm": 0.15920144319534302,
+      "learning_rate": 1.1229839215686294e-06,
+      "loss": 0.7599,
+      "step": 2050
+    },
+    {
+      "epoch": 0.7932229495571813,
+      "grad_norm": 0.15750554203987122,
+      "learning_rate": 1.0839095136509188e-06,
+      "loss": 0.7473,
+      "step": 2060
+    },
+    {
+      "epoch": 0.797073546399692,
+      "grad_norm": 0.183786541223526,
+      "learning_rate": 1.045444208397791e-06,
+      "loss": 0.7644,
+      "step": 2070
+    },
+    {
+      "epoch": 0.8009241432422025,
+      "grad_norm": 0.1620444655418396,
+      "learning_rate": 1.007593988643402e-06,
+      "loss": 0.7391,
+      "step": 2080
+    },
+    {
+      "epoch": 0.8047747400847132,
+      "grad_norm": 0.16444943845272064,
+      "learning_rate": 9.703647415524613e-07,
+      "loss": 0.7656,
+      "step": 2090
+    },
+    {
+      "epoch": 0.8086253369272237,
+      "grad_norm": 0.168411985039711,
+      "learning_rate": 9.337622577045485e-07,
+      "loss": 0.758,
+      "step": 2100
+    },
+    {
+      "epoch": 0.8124759337697343,
+      "grad_norm": 0.17231014370918274,
+      "learning_rate": 8.977922301934588e-07,
+      "loss": 0.7586,
+      "step": 2110
+    },
+    {
+      "epoch": 0.8163265306122449,
+      "grad_norm": 0.18492935597896576,
+      "learning_rate": 8.624602537417009e-07,
+      "loss": 0.7737,
+      "step": 2120
+    },
+    {
+      "epoch": 0.8201771274547555,
+      "grad_norm": 0.1726539134979248,
+      "learning_rate": 8.277718238303123e-07,
+      "loss": 0.756,
+      "step": 2130
+    },
+    {
+      "epoch": 0.8240277242972661,
+      "grad_norm": 0.177035391330719,
+      "learning_rate": 7.937323358440935e-07,
+      "loss": 0.7583,
+      "step": 2140
+    },
+    {
+      "epoch": 0.8278783211397767,
+      "grad_norm": 0.16525626182556152,
+      "learning_rate": 7.603470842324223e-07,
+      "loss": 0.7484,
+      "step": 2150
+    },
+    {
+      "epoch": 0.8317289179822872,
+      "grad_norm": 0.1678822934627533,
+      "learning_rate": 7.276212616857575e-07,
+      "loss": 0.7681,
+      "step": 2160
+    },
+    {
+      "epoch": 0.8355795148247979,
+      "grad_norm": 0.1699499487876892,
+      "learning_rate": 6.955599583279837e-07,
+      "loss": 0.7631,
+      "step": 2170
+    },
+    {
+      "epoch": 0.8394301116673084,
+      "grad_norm": 0.16490218043327332,
+      "learning_rate": 6.641681609246981e-07,
+      "loss": 0.7571,
+      "step": 2180
+    },
+    {
+      "epoch": 0.8432807085098191,
+      "grad_norm": 0.17616097629070282,
+      "learning_rate": 6.334507521075711e-07,
+      "loss": 0.7598,
+      "step": 2190
+    },
+    {
+      "epoch": 0.8471313053523296,
+      "grad_norm": 0.18033073842525482,
+      "learning_rate": 6.034125096149179e-07,
+      "loss": 0.7613,
+      "step": 2200
+    },
+    {
+      "epoch": 0.8509819021948402,
+      "grad_norm": 0.16474054753780365,
+      "learning_rate": 5.740581055485722e-07,
+      "loss": 0.7324,
+      "step": 2210
+    },
+    {
+      "epoch": 0.8548324990373508,
+      "grad_norm": 0.16352108120918274,
+      "learning_rate": 5.453921056471961e-07,
+      "loss": 0.7651,
+      "step": 2220
+    },
+    {
+      "epoch": 0.8586830958798614,
+      "grad_norm": 0.1876058727502823,
+      "learning_rate": 5.174189685761288e-07,
+      "loss": 0.758,
+      "step": 2230
+    },
+    {
+      "epoch": 0.862533692722372,
+      "grad_norm": 0.16764837503433228,
+      "learning_rate": 4.901430452338979e-07,
+      "loss": 0.7696,
+      "step": 2240
+    },
+    {
+      "epoch": 0.8663842895648826,
+      "grad_norm": 0.17329858243465424,
+      "learning_rate": 4.6356857807548286e-07,
+      "loss": 0.7585,
+      "step": 2250
+    },
+    {
+      "epoch": 0.8702348864073931,
+      "grad_norm": 0.1975717693567276,
+      "learning_rate": 4.376997004524541e-07,
+      "loss": 0.7406,
+      "step": 2260
+    },
+    {
+      "epoch": 0.8740854832499038,
+      "grad_norm": 0.1702103614807129,
+      "learning_rate": 4.12540435970073e-07,
+      "loss": 0.7602,
+      "step": 2270
+    },
+    {
+      "epoch": 0.8779360800924143,
+      "grad_norm": 0.16247408092021942,
+      "learning_rate": 3.880946978614703e-07,
+      "loss": 0.7621,
+      "step": 2280
+    },
+    {
+      "epoch": 0.881786676934925,
+      "grad_norm": 0.16893579065799713,
+      "learning_rate": 3.643662883789878e-07,
+      "loss": 0.7687,
+      "step": 2290
+    },
+    {
+      "epoch": 0.8856372737774355,
+      "grad_norm": 0.163793683052063,
+      "learning_rate": 3.41358898202781e-07,
+      "loss": 0.7493,
+      "step": 2300
+    },
+    {
+      "epoch": 0.889487870619946,
+      "grad_norm": 0.16484107077121735,
+      "learning_rate": 3.190761058667741e-07,
+      "loss": 0.7614,
+      "step": 2310
+    },
+    {
+      "epoch": 0.8933384674624567,
+      "grad_norm": 0.16285492479801178,
+      "learning_rate": 2.9752137720206463e-07,
+      "loss": 0.764,
+      "step": 2320
+    },
+    {
+      "epoch": 0.8971890643049673,
+      "grad_norm": 0.1753949224948883,
+      "learning_rate": 2.7669806479785067e-07,
+      "loss": 0.773,
+      "step": 2330
+    },
+    {
+      "epoch": 0.9010396611474779,
+      "grad_norm": 0.16498513519763947,
+      "learning_rate": 2.566094074799763e-07,
+      "loss": 0.7662,
+      "step": 2340
+    },
+    {
+      "epoch": 0.9048902579899885,
+      "grad_norm": 0.17592117190361023,
+      "learning_rate": 2.3725852980716557e-07,
+      "loss": 0.7439,
+      "step": 2350
+    },
+    {
+      "epoch": 0.908740854832499,
+      "grad_norm": 0.16625143587589264,
+      "learning_rate": 2.1864844158503983e-07,
+      "loss": 0.7599,
+      "step": 2360
+    },
+    {
+      "epoch": 0.9125914516750097,
+      "grad_norm": 0.18389025330543518,
+      "learning_rate": 2.007820373979702e-07,
+      "loss": 0.752,
+      "step": 2370
+    },
+    {
+      "epoch": 0.9164420485175202,
+      "grad_norm": 0.16617615520954132,
+      "learning_rate": 1.8366209615886532e-07,
+      "loss": 0.7626,
+      "step": 2380
+    },
+    {
+      "epoch": 0.9202926453600309,
+      "grad_norm": 0.17198534309864044,
+      "learning_rate": 1.6729128067693613e-07,
+      "loss": 0.7726,
+      "step": 2390
+    },
+    {
+      "epoch": 0.9241432422025414,
+      "grad_norm": 0.19071416556835175,
+      "learning_rate": 1.5167213724353426e-07,
+      "loss": 0.7683,
+      "step": 2400
+    },
+    {
+      "epoch": 0.927993839045052,
+      "grad_norm": 0.16362757980823517,
+      "learning_rate": 1.3680709523610058e-07,
+      "loss": 0.7457,
+      "step": 2410
+    },
+    {
+      "epoch": 0.9318444358875626,
+      "grad_norm": 0.16056659817695618,
+      "learning_rate": 1.2269846674030418e-07,
+      "loss": 0.777,
+      "step": 2420
+    },
+    {
+      "epoch": 0.9356950327300732,
+      "grad_norm": 0.1646297574043274,
+      "learning_rate": 1.0934844619042595e-07,
+      "loss": 0.7698,
+      "step": 2430
+    },
+    {
+      "epoch": 0.9395456295725837,
+      "grad_norm": 0.17043940722942352,
+      "learning_rate": 9.675911002803961e-08,
+      "loss": 0.7607,
+      "step": 2440
+    },
+    {
+      "epoch": 0.9433962264150944,
+      "grad_norm": 0.1810024529695511,
+      "learning_rate": 8.493241637904381e-08,
+      "loss": 0.7492,
+      "step": 2450
+    },
+    {
+      "epoch": 0.9472468232576049,
+      "grad_norm": 0.17686310410499573,
+      "learning_rate": 7.387020474909689e-08,
+      "loss": 0.7749,
+      "step": 2460
+    },
+    {
+      "epoch": 0.9510974201001156,
+      "grad_norm": 0.1773536652326584,
+      "learning_rate": 6.357419573750856e-08,
+      "loss": 0.7713,
+      "step": 2470
+    },
+    {
+      "epoch": 0.9549480169426261,
+      "grad_norm": 0.17506557703018188,
+      "learning_rate": 5.4045990769613365e-08,
+      "loss": 0.7568,
+      "step": 2480
+    },
+    {
+      "epoch": 0.9587986137851366,
+      "grad_norm": 0.17210182547569275,
+      "learning_rate": 4.5287071847694434e-08,
+      "loss": 0.7586,
+      "step": 2490
+    },
+    {
+      "epoch": 0.9626492106276473,
+      "grad_norm": 0.16752995550632477,
+      "learning_rate": 3.7298801320467817e-08,
+      "loss": 0.7591,
+      "step": 2500
+    },
+    {
+      "epoch": 0.9664998074701578,
+      "grad_norm": 0.16701336205005646,
+      "learning_rate": 3.0082421671192576e-08,
+      "loss": 0.7583,
+      "step": 2510
+    },
+    {
+      "epoch": 0.9703504043126685,
+      "grad_norm": 0.17360830307006836,
+      "learning_rate": 2.3639055324410355e-08,
+      "loss": 0.7713,
+      "step": 2520
+    },
+    {
+      "epoch": 0.974201001155179,
+      "grad_norm": 0.17269811034202576,
+      "learning_rate": 1.7969704471369477e-08,
+      "loss": 0.7536,
+      "step": 2530
+    },
+    {
+      "epoch": 0.9780515979976896,
+      "grad_norm": 0.16033458709716797,
+      "learning_rate": 1.3075250914142413e-08,
+      "loss": 0.7531,
+      "step": 2540
+    },
+    {
+      "epoch": 0.9819021948402002,
+      "grad_norm": 0.15802381932735443,
+      "learning_rate": 8.956455928476049e-09,
+      "loss": 0.7698,
+      "step": 2550
+    },
+    {
+      "epoch": 0.9857527916827108,
+      "grad_norm": 0.16353990137577057,
+      "learning_rate": 5.6139601453786275e-09,
+      "loss": 0.7547,
+      "step": 2560
+    },
+    {
+      "epoch": 0.9896033885252215,
+      "grad_norm": 0.1667587161064148,
+      "learning_rate": 3.0482834514805602e-09,
+      "loss": 0.7501,
+      "step": 2570
+    },
+    {
+      "epoch": 0.993453985367732,
+      "grad_norm": 0.1627393364906311,
+      "learning_rate": 1.2598249081713366e-09,
+      "loss": 0.7424,
+      "step": 2580
+    },
+    {
+      "epoch": 0.9973045822102425,
+      "grad_norm": 0.1769946962594986,
+      "learning_rate": 2.488626895302826e-10,
+      "loss": 0.7731,
+      "step": 2590
+    },
+    {
+      "epoch": 1.0,
+      "step": 2597,
+      "total_flos": 6.733660633738248e+18,
+      "train_loss": 0.7762898409141684,
+      "train_runtime": 14892.211,
+      "train_samples_per_second": 44.648,
+      "train_steps_per_second": 0.174
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2597,
+  "num_input_tokens_seen": 584317136,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.733660633738248e+18,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

continuous/smollm2/webssl_dino300m_full2b_224/finetune/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76fd4602c06934fed25f01a2a52cb01b00c73b646c2f123a6413d0d574d1a020
+size 6264

continuous/smollm2/webssl_dino300m_full2b_224/pretrain/config.json

@@ -0,0 +1,70 @@
+{
+  "add_faster_video": false,
+  "add_time_instruction": false,
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "faster_token_stride": 10,
+  "force_sample": false,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_aspect_ratio": "square",
+  "image_crop_resolution": null,
+  "image_grid_pinpoints": null,
+  "image_split_resolution": null,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mm_hidden_size": 1024,
+  "mm_newline_position": "grid",
+  "mm_patch_merge_type": "flat",
+  "mm_projector_lr": null,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_resampler_type": null,
+  "mm_spatial_pool_mode": "bilinear",
+  "mm_spatial_pool_stride": null,
+  "mm_tunable_parts": "mm_mlp_adapter",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -1,
+  "mm_vision_tower": "hf:/cache/ckpt/download/tokenizer/continuous/webssl/webssl-dino300m-full2b-224",
+  "mm_vision_tower_lr": null,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "pos_skipping_range": 4096,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 130000,
+  "tie_word_embeddings": true,
+  "tokenizer_model_max_length": 8192,
+  "tokenizer_padding_side": "right",
+  "transformers.js_config": {
+    "dtype": "q4",
+    "kv_cache_dtype": {
+      "fp16": "float16",
+      "q4f16": "float16"
+    },
+    "use_external_data_format": {
+      "model.onnx": true,
+      "model_fp16.onnx": true
+    }
+  },
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "use_mm_proj": true,
+  "use_pos_skipping": false,
+  "vision_tower_pretrained": null,
+  "vocab_size": 49152
+}

continuous/smollm2/webssl_dino300m_full2b_224/pretrain/mm_projector.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9706eeaa37383c16fdfae4050b62d06231a6428eff94fcf554081a28e3e166c
+size 25184312

continuous/smollm2/webssl_dino300m_full2b_224/pretrain/trainer_state.json

@@ -0,0 +1,421 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 545,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01834862385321101,
+      "grad_norm": 1.3608818054199219,
+      "learning_rate": 0.00032142857142857147,
+      "loss": 3.7328,
+      "step": 10
+    },
+    {
+      "epoch": 0.03669724770642202,
+      "grad_norm": 0.17641086876392365,
+      "learning_rate": 0.0006785714285714287,
+      "loss": 2.9274,
+      "step": 20
+    },
+    {
+      "epoch": 0.05504587155963303,
+      "grad_norm": 0.09021255373954773,
+      "learning_rate": 0.0009999907688176173,
+      "loss": 2.7821,
+      "step": 30
+    },
+    {
+      "epoch": 0.07339449541284404,
+      "grad_norm": 0.10445146262645721,
+      "learning_rate": 0.0009988834393115766,
+      "loss": 2.7041,
+      "step": 40
+    },
+    {
+      "epoch": 0.09174311926605505,
+      "grad_norm": 0.08677558600902557,
+      "learning_rate": 0.0009959345572940772,
+      "loss": 2.6567,
+      "step": 50
+    },
+    {
+      "epoch": 0.11009174311926606,
+      "grad_norm": 0.11340591311454773,
+      "learning_rate": 0.0009911550081155983,
+      "loss": 2.6073,
+      "step": 60
+    },
+    {
+      "epoch": 0.12844036697247707,
+      "grad_norm": 0.13171449303627014,
+      "learning_rate": 0.0009845624347566433,
+      "loss": 2.5743,
+      "step": 70
+    },
+    {
+      "epoch": 0.14678899082568808,
+      "grad_norm": 0.13868063688278198,
+      "learning_rate": 0.0009761811727013547,
+      "loss": 2.5433,
+      "step": 80
+    },
+    {
+      "epoch": 0.1651376146788991,
+      "grad_norm": 0.1355467438697815,
+      "learning_rate": 0.0009660421601067666,
+      "loss": 2.5203,
+      "step": 90
+    },
+    {
+      "epoch": 0.1834862385321101,
+      "grad_norm": 0.13812386989593506,
+      "learning_rate": 0.0009541828235992926,
+      "loss": 2.5084,
+      "step": 100
+    },
+    {
+      "epoch": 0.2018348623853211,
+      "grad_norm": 0.09742917120456696,
+      "learning_rate": 0.0009406469401200151,
+      "loss": 2.4841,
+      "step": 110
+    },
+    {
+      "epoch": 0.22018348623853212,
+      "grad_norm": 0.09250123053789139,
+      "learning_rate": 0.0009254844753287492,
+      "loss": 2.4761,
+      "step": 120
+    },
+    {
+      "epoch": 0.23853211009174313,
+      "grad_norm": 0.108077771961689,
+      "learning_rate": 0.0009087513991633924,
+      "loss": 2.4561,
+      "step": 130
+    },
+    {
+      "epoch": 0.25688073394495414,
+      "grad_norm": 0.09849868714809418,
+      "learning_rate": 0.0008905094792353917,
+      "loss": 2.4453,
+      "step": 140
+    },
+    {
+      "epoch": 0.27522935779816515,
+      "grad_norm": 0.09327493607997894,
+      "learning_rate": 0.0008708260528239789,
+      "loss": 2.4509,
+      "step": 150
+    },
+    {
+      "epoch": 0.29357798165137616,
+      "grad_norm": 0.08948465436697006,
+      "learning_rate": 0.0008497737783108237,
+      "loss": 2.4389,
+      "step": 160
+    },
+    {
+      "epoch": 0.3119266055045872,
+      "grad_norm": 0.07853515446186066,
+      "learning_rate": 0.0008274303669726426,
+      "loss": 2.4342,
+      "step": 170
+    },
+    {
+      "epoch": 0.3302752293577982,
+      "grad_norm": 0.10086923092603683,
+      "learning_rate": 0.0008038782961218137,
+      "loss": 2.4229,
+      "step": 180
+    },
+    {
+      "epoch": 0.3486238532110092,
+      "grad_norm": 0.10101193189620972,
+      "learning_rate": 0.000779204504653894,
+      "loss": 2.4191,
+      "step": 190
+    },
+    {
+      "epoch": 0.3669724770642202,
+      "grad_norm": 0.08335606008768082,
+      "learning_rate": 0.0007535000721258791,
+      "loss": 2.4001,
+      "step": 200
+    },
+    {
+      "epoch": 0.3853211009174312,
+      "grad_norm": 0.0866546630859375,
+      "learning_rate": 0.00072685988254984,
+      "loss": 2.4021,
+      "step": 210
+    },
+    {
+      "epoch": 0.4036697247706422,
+      "grad_norm": 0.09501807391643524,
+      "learning_rate": 0.0006993822741429906,
+      "loss": 2.4083,
+      "step": 220
+    },
+    {
+      "epoch": 0.42201834862385323,
+      "grad_norm": 0.08033736795186996,
+      "learning_rate": 0.0006711686763270818,
+      "loss": 2.4051,
+      "step": 230
+    },
+    {
+      "epoch": 0.44036697247706424,
+      "grad_norm": 0.08737606555223465,
+      "learning_rate": 0.0006423232353170831,
+      "loss": 2.397,
+      "step": 240
+    },
+    {
+      "epoch": 0.45871559633027525,
+      "grad_norm": 0.08756648749113083,
+      "learning_rate": 0.0006129524296812335,
+      "loss": 2.3887,
+      "step": 250
+    },
+    {
+      "epoch": 0.47706422018348627,
+      "grad_norm": 0.08900777250528336,
+      "learning_rate": 0.0005831646772915651,
+      "loss": 2.3979,
+      "step": 260
+    },
+    {
+      "epoch": 0.4954128440366973,
+      "grad_norm": 0.08272869139909744,
+      "learning_rate": 0.0005530699351157809,
+      "loss": 2.3775,
+      "step": 270
+    },
+    {
+      "epoch": 0.5137614678899083,
+      "grad_norm": 0.1028878390789032,
+      "learning_rate": 0.0005227792933277943,
+      "loss": 2.3833,
+      "step": 280
+    },
+    {
+      "epoch": 0.5321100917431193,
+      "grad_norm": 0.08761882781982422,
+      "learning_rate": 0.0004924045652352092,
+      "loss": 2.3676,
+      "step": 290
+    },
+    {
+      "epoch": 0.5504587155963303,
+      "grad_norm": 0.07924611121416092,
+      "learning_rate": 0.0004620578745374633,
+      "loss": 2.3677,
+      "step": 300
+    },
+    {
+      "epoch": 0.5688073394495413,
+      "grad_norm": 0.11063561588525772,
+      "learning_rate": 0.0004318512414382059,
+      "loss": 2.3761,
+      "step": 310
+    },
+    {
+      "epoch": 0.5871559633027523,
+      "grad_norm": 0.08376167714595795,
+      "learning_rate": 0.0004018961691397148,
+      "loss": 2.3758,
+      "step": 320
+    },
+    {
+      "epoch": 0.6055045871559633,
+      "grad_norm": 0.10660155862569809,
+      "learning_rate": 0.0003723032322457458,
+      "loss": 2.3593,
+      "step": 330
+    },
+    {
+      "epoch": 0.6238532110091743,
+      "grad_norm": 0.09873966127634048,
+      "learning_rate": 0.0003431816685921625,
+      "loss": 2.3656,
+      "step": 340
+    },
+    {
+      "epoch": 0.6422018348623854,
+      "grad_norm": 0.07969395071268082,
+      "learning_rate": 0.0003146389760120416,
+      "loss": 2.3664,
+      "step": 350
+    },
+    {
+      "epoch": 0.6605504587155964,
+      "grad_norm": 0.09483855962753296,
+      "learning_rate": 0.00028678051552373485,
+      "loss": 2.3694,
+      "step": 360
+    },
+    {
+      "epoch": 0.6788990825688074,
+      "grad_norm": 0.0823395624756813,
+      "learning_rate": 0.0002597091224066581,
+      "loss": 2.3596,
+      "step": 370
+    },
+    {
+      "epoch": 0.6972477064220184,
+      "grad_norm": 0.07977040112018585,
+      "learning_rate": 0.0002335247266004629,
+      "loss": 2.3693,
+      "step": 380
+    },
+    {
+      "epoch": 0.7155963302752294,
+      "grad_norm": 0.08550091832876205,
+      "learning_rate": 0.000208323983828828,
+      "loss": 2.353,
+      "step": 390
+    },
+    {
+      "epoch": 0.7339449541284404,
+      "grad_norm": 0.07875966280698776,
+      "learning_rate": 0.0001841999188095224,
+      "loss": 2.355,
+      "step": 400
+    },
+    {
+      "epoch": 0.7522935779816514,
+      "grad_norm": 0.08284778147935867,
+      "learning_rate": 0.00016124158186777676,
+      "loss": 2.3515,
+      "step": 410
+    },
+    {
+      "epoch": 0.7706422018348624,
+      "grad_norm": 0.08487748354673386,
+      "learning_rate": 0.0001395337202205211,
+      "loss": 2.3468,
+      "step": 420
+    },
+    {
+      "epoch": 0.7889908256880734,
+      "grad_norm": 0.0781325250864029,
+      "learning_rate": 0.00011915646514489292,
+      "loss": 2.3519,
+      "step": 430
+    },
+    {
+      "epoch": 0.8073394495412844,
+      "grad_norm": 0.09572076797485352,
+      "learning_rate": 0.00010018503618578818,
+      "loss": 2.3574,
+      "step": 440
+    },
+    {
+      "epoch": 0.8256880733944955,
+      "grad_norm": 0.08173124492168427,
+      "learning_rate": 8.268946349432583e-05,
+      "loss": 2.3537,
+      "step": 450
+    },
+    {
+      "epoch": 0.8440366972477065,
+      "grad_norm": 0.0810076892375946,
+      "learning_rate": 6.673432932217117e-05,
+      "loss": 2.3417,
+      "step": 460
+    },
+    {
+      "epoch": 0.8623853211009175,
+      "grad_norm": 0.0846480131149292,
+      "learning_rate": 5.2378529625954684e-05,
+      "loss": 2.3375,
+      "step": 470
+    },
+    {
+      "epoch": 0.8807339449541285,
+      "grad_norm": 0.08047457039356232,
+      "learning_rate": 3.9675056661785556e-05,
+      "loss": 2.3424,
+      "step": 480
+    },
+    {
+      "epoch": 0.8990825688073395,
+      "grad_norm": 0.0802660807967186,
+      "learning_rate": 2.8670803372379538e-05,
+      "loss": 2.3458,
+      "step": 490
+    },
+    {
+      "epoch": 0.9174311926605505,
+      "grad_norm": 0.08082406222820282,
+      "learning_rate": 1.9406390288876586e-05,
+      "loss": 2.3516,
+      "step": 500
+    },
+    {
+      "epoch": 0.9357798165137615,
+      "grad_norm": 0.08159183710813522,
+      "learning_rate": 1.191601558631461e-05,
+      "loss": 2.3403,
+      "step": 510
+    },
+    {
+      "epoch": 0.9541284403669725,
+      "grad_norm": 0.07743487507104874,
+      "learning_rate": 6.2273288462559305e-06,
+      "loss": 2.3363,
+      "step": 520
+    },
+    {
+      "epoch": 0.9724770642201835,
+      "grad_norm": 0.07765097916126251,
+      "learning_rate": 2.361328992552314e-06,
+      "loss": 2.3379,
+      "step": 530
+    },
+    {
+      "epoch": 0.9908256880733946,
+      "grad_norm": 0.07882232964038849,
+      "learning_rate": 3.322867770044202e-07,
+      "loss": 2.3422,
+      "step": 540
+    },
+    {
+      "epoch": 1.0,
+      "step": 545,
+      "total_flos": 3.994002262358753e+17,
+      "train_loss": 2.450589505466846,
+      "train_runtime": 1647.1269,
+      "train_samples_per_second": 338.849,
+      "train_steps_per_second": 0.331
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 545,
+  "num_input_tokens_seen": 34658176,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.994002262358753e+17,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

continuous/smollm2/webssl_mae1b_full2b_224/finetune/chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

continuous/smollm2/webssl_mae1b_full2b_224/finetune/config.json

@@ -0,0 +1,70 @@
+{
+  "add_faster_video": false,
+  "add_time_instruction": false,
+  "architectures": [
+    "LlavaLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "faster_token_stride": 10,
+  "force_sample": false,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_aspect_ratio": "square",
+  "image_crop_resolution": null,
+  "image_grid_pinpoints": null,
+  "image_split_resolution": null,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mm_hidden_size": 1536,
+  "mm_newline_position": "grid",
+  "mm_patch_merge_type": "flat",
+  "mm_projector_lr": 0.0001,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_resampler_type": null,
+  "mm_spatial_pool_mode": "bilinear",
+  "mm_spatial_pool_stride": null,
+  "mm_tunable_parts": "mm_mlp_adapter,mm_language_model",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -1,
+  "mm_vision_tower": "hf:/cache/ckpt/download/tokenizer/continuous/webssl/webssl-mae1b-full2b-224",
+  "mm_vision_tower_lr": null,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "pos_skipping_range": 4096,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 130000,
+  "tie_word_embeddings": true,
+  "tokenizer_model_max_length": 8192,
+  "tokenizer_padding_side": "right",
+  "transformers.js_config": {
+    "dtype": "q4",
+    "kv_cache_dtype": {
+      "fp16": "float16",
+      "q4f16": "float16"
+    },
+    "use_external_data_format": {
+      "model.onnx": true,
+      "model_fp16.onnx": true
+    }
+  },
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "use_mm_proj": true,
+  "use_pos_skipping": false,
+  "vision_tower_pretrained": null,
+  "vocab_size": 49152
+}

continuous/smollm2/webssl_mae1b_full2b_224/finetune/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    2
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.57.6"
+}

continuous/smollm2/webssl_mae1b_full2b_224/finetune/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2427f52f1bbcbf68c5a4fd30b0a7c5102bf4cd870c652158248a88758a8abb7
+size 4993122248

continuous/smollm2/webssl_mae1b_full2b_224/finetune/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:388f7798292e5d9641f8f8b0e9e9d8b754a6b5a1ced35c0fa2d031ecdc5a0cda
+size 934307456

continuous/smollm2/webssl_mae1b_full2b_224/finetune/model.safetensors.index.json

@@ -0,0 +1,879 @@
+{
+  "metadata": {
+    "total_parameters": 2855646208,
+    "total_size": 5927307264
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.mm_projector.0.bias": "model-00002-of-00002.safetensors",
+    "model.mm_projector.0.weight": "model-00002-of-00002.safetensors",
+    "model.mm_projector.2.bias": "model-00002-of-00002.safetensors",
+    "model.mm_projector.2.weight": "model-00002-of-00002.safetensors",
+    "model.norm.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.embeddings.cls_token": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.embeddings.patch_embeddings.projection.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.embeddings.patch_embeddings.projection.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.embeddings.position_embeddings": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.0.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.0.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.0.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.0.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.0.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.0.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.0.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.0.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.0.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.0.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.0.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.0.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.0.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.0.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.0.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.0.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.1.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.1.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.1.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.1.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.1.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.1.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.1.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.1.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.1.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.1.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.1.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.1.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.1.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.1.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.1.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.1.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.10.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.10.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.10.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.10.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.10.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.10.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.10.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.10.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.10.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.10.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.10.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.10.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.10.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.10.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.10.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.10.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.11.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.11.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.11.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.11.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.11.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.11.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.11.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.11.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.11.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.11.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.11.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.11.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.11.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.11.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.11.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.11.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.12.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.12.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.12.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.12.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.12.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.12.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.12.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.12.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.12.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.12.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.12.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.12.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.12.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.12.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.12.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.12.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.13.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.13.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.13.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.13.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.13.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.13.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.13.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.13.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.13.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.13.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.13.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.13.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.13.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.13.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.13.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.13.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.14.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.14.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.14.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.14.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.14.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.14.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.14.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.14.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.14.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.14.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.14.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.14.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.14.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.14.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.14.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.14.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.15.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.15.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.15.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.15.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.15.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.15.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.15.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.15.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.15.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.15.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.15.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.15.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.15.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.15.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.15.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.15.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.16.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.16.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.16.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.16.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.16.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.16.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.16.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.16.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.16.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.16.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.16.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.16.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.16.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.16.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.16.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.16.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.17.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.17.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.17.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.17.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.17.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.17.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.17.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.17.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.17.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.17.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.17.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.17.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.17.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.17.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.17.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.17.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.18.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.18.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.18.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.18.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.18.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.18.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.18.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.18.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.18.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.18.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.18.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.18.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.18.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.18.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.18.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.18.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.19.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.19.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.19.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.19.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.19.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.19.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.19.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.19.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.19.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.19.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.19.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.19.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.19.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.19.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.19.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.19.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.2.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.2.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.2.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.2.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.2.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.2.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.2.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.2.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.2.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.2.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.2.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.2.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.2.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.2.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.2.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.2.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.20.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.20.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.20.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.20.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.20.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.20.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.20.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.20.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.20.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.20.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.20.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.20.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.20.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.20.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.20.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.20.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.21.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.21.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.21.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.21.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.21.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.21.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.21.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.21.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.21.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.21.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.21.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.21.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.21.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.21.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.21.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.21.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.22.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.22.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.22.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.22.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.22.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.22.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.22.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.22.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.22.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.22.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.22.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.22.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.22.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.22.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.22.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.22.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.23.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.23.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.23.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.23.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.23.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.23.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.23.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.23.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.23.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.23.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.23.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.23.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.23.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.23.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.23.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.23.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.24.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.24.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.24.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.24.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.24.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.24.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.24.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.24.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.24.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.24.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.24.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.24.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.24.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.24.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.24.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.24.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.25.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.25.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.25.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.25.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.25.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.25.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.25.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.25.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.25.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.25.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.25.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.25.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.25.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.25.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.25.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.25.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.26.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.26.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.26.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.26.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.26.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.26.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.26.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.26.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.26.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.26.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.26.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.26.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.26.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.26.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.26.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.26.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.27.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.27.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.27.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.27.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.27.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.27.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.27.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.27.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.27.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.27.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.27.layernorm_after.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.27.layernorm_after.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.27.layernorm_before.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.27.layernorm_before.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.27.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.27.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.28.attention.attention.key.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.28.attention.attention.key.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.28.attention.attention.query.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.28.attention.attention.query.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.28.attention.attention.value.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.28.attention.attention.value.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.28.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.28.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.28.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.28.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.28.layernorm_after.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.28.layernorm_after.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.28.layernorm_before.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.28.layernorm_before.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.28.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.28.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.29.attention.attention.key.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.29.attention.attention.key.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.29.attention.attention.query.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.29.attention.attention.query.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.29.attention.attention.value.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.29.attention.attention.value.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.29.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.29.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.29.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.29.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.29.layernorm_after.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.29.layernorm_after.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.29.layernorm_before.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.29.layernorm_before.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.29.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.29.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.3.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.3.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.3.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.3.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.3.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.3.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.3.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.3.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.3.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.3.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.3.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.3.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.3.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.3.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.3.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.3.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.30.attention.attention.key.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.30.attention.attention.key.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.30.attention.attention.query.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.30.attention.attention.query.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.30.attention.attention.value.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.30.attention.attention.value.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.30.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.30.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.30.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.30.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.30.layernorm_after.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.30.layernorm_after.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.30.layernorm_before.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.30.layernorm_before.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.30.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.30.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.31.attention.attention.key.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.31.attention.attention.key.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.31.attention.attention.query.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.31.attention.attention.query.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.31.attention.attention.value.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.31.attention.attention.value.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.31.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.31.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.31.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.31.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.31.layernorm_after.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.31.layernorm_after.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.31.layernorm_before.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.31.layernorm_before.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.31.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.31.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.32.attention.attention.key.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.32.attention.attention.key.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.32.attention.attention.query.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.32.attention.attention.query.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.32.attention.attention.value.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.32.attention.attention.value.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.32.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.32.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.32.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.32.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.32.layernorm_after.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.32.layernorm_after.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.32.layernorm_before.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.32.layernorm_before.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.32.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.32.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.33.attention.attention.key.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.33.attention.attention.key.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.33.attention.attention.query.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.33.attention.attention.query.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.33.attention.attention.value.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.33.attention.attention.value.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.33.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.33.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.33.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.33.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.33.layernorm_after.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.33.layernorm_after.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.33.layernorm_before.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.33.layernorm_before.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.33.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.33.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.34.attention.attention.key.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.34.attention.attention.key.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.34.attention.attention.query.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.34.attention.attention.query.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.34.attention.attention.value.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.34.attention.attention.value.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.34.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.34.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.34.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.34.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.34.layernorm_after.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.34.layernorm_after.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.34.layernorm_before.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.34.layernorm_before.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.34.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.34.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.35.attention.attention.key.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.35.attention.attention.key.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.35.attention.attention.query.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.35.attention.attention.query.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.35.attention.attention.value.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.35.attention.attention.value.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.35.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.35.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.35.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.35.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.35.layernorm_after.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.35.layernorm_after.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.35.layernorm_before.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.35.layernorm_before.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.35.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.35.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.36.attention.attention.key.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.36.attention.attention.key.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.36.attention.attention.query.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.36.attention.attention.query.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.36.attention.attention.value.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.36.attention.attention.value.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.36.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.36.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.36.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.36.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.36.layernorm_after.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.36.layernorm_after.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.36.layernorm_before.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.36.layernorm_before.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.36.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.36.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.37.attention.attention.key.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.37.attention.attention.key.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.37.attention.attention.query.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.37.attention.attention.query.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.37.attention.attention.value.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.37.attention.attention.value.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.37.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.37.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.37.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.37.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.37.layernorm_after.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.37.layernorm_after.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.37.layernorm_before.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.37.layernorm_before.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.37.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.37.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.38.attention.attention.key.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.38.attention.attention.key.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.38.attention.attention.query.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.38.attention.attention.query.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.38.attention.attention.value.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.38.attention.attention.value.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.38.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.38.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.38.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.38.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.38.layernorm_after.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.38.layernorm_after.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.38.layernorm_before.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.38.layernorm_before.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.38.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.38.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.39.attention.attention.key.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.39.attention.attention.key.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.39.attention.attention.query.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.39.attention.attention.query.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.39.attention.attention.value.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.39.attention.attention.value.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.39.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.39.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.39.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.39.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.39.layernorm_after.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.39.layernorm_after.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.39.layernorm_before.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.39.layernorm_before.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.39.output.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.39.output.dense.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.4.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.4.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.4.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.4.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.4.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.4.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.4.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.4.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.4.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.4.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.4.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.4.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.4.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.4.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.4.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.4.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.5.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.5.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.5.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.5.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.5.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.5.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.5.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.5.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.5.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.5.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.5.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.5.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.5.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.5.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.5.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.5.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.6.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.6.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.6.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.6.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.6.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.6.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.6.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.6.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.6.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.6.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.6.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.6.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.6.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.6.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.6.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.6.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.7.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.7.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.7.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.7.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.7.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.7.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.7.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.7.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.7.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.7.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.7.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.7.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.7.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.7.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.7.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.7.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.8.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.8.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.8.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.8.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.8.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.8.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.8.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.8.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.8.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.8.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.8.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.8.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.8.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.8.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.8.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.8.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.9.attention.attention.key.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.9.attention.attention.key.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.9.attention.attention.query.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.9.attention.attention.query.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.9.attention.attention.value.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.9.attention.attention.value.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.9.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.9.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.9.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.9.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.9.layernorm_after.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.9.layernorm_after.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.9.layernorm_before.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.9.layernorm_before.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.9.output.dense.bias": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.encoder.layer.9.output.dense.weight": "model-00001-of-00002.safetensors",
+    "model.vision_tower.vision_tower.layernorm.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.pooler.dense.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.pooler.dense.weight": "model-00002-of-00002.safetensors"
+  }
+}

continuous/smollm2/webssl_mae1b_full2b_224/finetune/special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

continuous/smollm2/webssl_mae1b_full2b_224/finetune/tokenizer_config.json

@@ -0,0 +1,155 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

continuous/smollm2/webssl_mae1b_full2b_224/finetune/trainer_state.json

@@ -0,0 +1,1856 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 2597,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.003850596842510589,
+      "grad_norm": 2.8468148708343506,
+      "learning_rate": 1.153846153846154e-06,
+      "loss": 1.2255,
+      "step": 10
+    },
+    {
+      "epoch": 0.007701193685021178,
+      "grad_norm": 2.1306209564208984,
+      "learning_rate": 2.435897435897436e-06,
+      "loss": 1.1431,
+      "step": 20
+    },
+    {
+      "epoch": 0.011551790527531768,
+      "grad_norm": 2.0805108547210693,
+      "learning_rate": 3.7179487179487184e-06,
+      "loss": 1.0592,
+      "step": 30
+    },
+    {
+      "epoch": 0.015402387370042356,
+      "grad_norm": 1.3975688219070435,
+      "learning_rate": 5e-06,
+      "loss": 1.0175,
+      "step": 40
+    },
+    {
+      "epoch": 0.019252984212552945,
+      "grad_norm": 1.3737074136734009,
+      "learning_rate": 6.282051282051282e-06,
+      "loss": 0.9994,
+      "step": 50
+    },
+    {
+      "epoch": 0.023103581055063535,
+      "grad_norm": 0.32066357135772705,
+      "learning_rate": 7.564102564102564e-06,
+      "loss": 0.9243,
+      "step": 60
+    },
+    {
+      "epoch": 0.026954177897574125,
+      "grad_norm": 0.2596486210823059,
+      "learning_rate": 8.846153846153847e-06,
+      "loss": 0.8767,
+      "step": 70
+    },
+    {
+      "epoch": 0.030804774740084712,
+      "grad_norm": 0.24903124570846558,
+      "learning_rate": 9.999996111488723e-06,
+      "loss": 0.887,
+      "step": 80
+    },
+    {
+      "epoch": 0.0346553715825953,
+      "grad_norm": 0.22882354259490967,
+      "learning_rate": 9.999529497453782e-06,
+      "loss": 0.8718,
+      "step": 90
+    },
+    {
+      "epoch": 0.03850596842510589,
+      "grad_norm": 0.21190734207630157,
+      "learning_rate": 9.998285264324193e-06,
+      "loss": 0.84,
+      "step": 100
+    },
+    {
+      "epoch": 0.04235656526761648,
+      "grad_norm": 0.2222384363412857,
+      "learning_rate": 9.996263605626052e-06,
+      "loss": 0.8297,
+      "step": 110
+    },
+    {
+      "epoch": 0.04620716211012707,
+      "grad_norm": 0.2265671193599701,
+      "learning_rate": 9.99346483580503e-06,
+      "loss": 0.849,
+      "step": 120
+    },
+    {
+      "epoch": 0.05005775895263766,
+      "grad_norm": 0.22575761377811432,
+      "learning_rate": 9.989889390177467e-06,
+      "loss": 0.8263,
+      "step": 130
+    },
+    {
+      "epoch": 0.05390835579514825,
+      "grad_norm": 0.23046401143074036,
+      "learning_rate": 9.985537824862647e-06,
+      "loss": 0.8386,
+      "step": 140
+    },
+    {
+      "epoch": 0.05775895263765884,
+      "grad_norm": 0.2509758174419403,
+      "learning_rate": 9.980410816696317e-06,
+      "loss": 0.8167,
+      "step": 150
+    },
+    {
+      "epoch": 0.061609549480169425,
+      "grad_norm": 0.2236732840538025,
+      "learning_rate": 9.97450916312541e-06,
+      "loss": 0.8214,
+      "step": 160
+    },
+    {
+      "epoch": 0.06546014632268002,
+      "grad_norm": 0.20413415133953094,
+      "learning_rate": 9.967833782084004e-06,
+      "loss": 0.8212,
+      "step": 170
+    },
+    {
+      "epoch": 0.0693107431651906,
+      "grad_norm": 0.22238512337207794,
+      "learning_rate": 9.960385711850554e-06,
+      "loss": 0.8551,
+      "step": 180
+    },
+    {
+      "epoch": 0.07316134000770119,
+      "grad_norm": 0.23155397176742554,
+      "learning_rate": 9.952166110886394e-06,
+      "loss": 0.8423,
+      "step": 190
+    },
+    {
+      "epoch": 0.07701193685021178,
+      "grad_norm": 0.24704472720623016,
+      "learning_rate": 9.943176257655567e-06,
+      "loss": 0.8409,
+      "step": 200
+    },
+    {
+      "epoch": 0.08086253369272237,
+      "grad_norm": 0.22274954617023468,
+      "learning_rate": 9.933417550425952e-06,
+      "loss": 0.8261,
+      "step": 210
+    },
+    {
+      "epoch": 0.08471313053523297,
+      "grad_norm": 0.22320061922073364,
+      "learning_rate": 9.9228915070518e-06,
+      "loss": 0.819,
+      "step": 220
+    },
+    {
+      "epoch": 0.08856372737774355,
+      "grad_norm": 0.22787252068519592,
+      "learning_rate": 9.911599764737632e-06,
+      "loss": 0.8129,
+      "step": 230
+    },
+    {
+      "epoch": 0.09241432422025414,
+      "grad_norm": 0.2141425460577011,
+      "learning_rate": 9.899544079783606e-06,
+      "loss": 0.8067,
+      "step": 240
+    },
+    {
+      "epoch": 0.09626492106276473,
+      "grad_norm": 0.20911435782909393,
+      "learning_rate": 9.886726327312338e-06,
+      "loss": 0.8245,
+      "step": 250
+    },
+    {
+      "epoch": 0.10011551790527531,
+      "grad_norm": 0.20520664751529694,
+      "learning_rate": 9.87314850097724e-06,
+      "loss": 0.8238,
+      "step": 260
+    },
+    {
+      "epoch": 0.1039661147477859,
+      "grad_norm": 0.20591966807842255,
+      "learning_rate": 9.858812712652452e-06,
+      "loss": 0.8047,
+      "step": 270
+    },
+    {
+      "epoch": 0.1078167115902965,
+      "grad_norm": 0.2001739740371704,
+      "learning_rate": 9.843721192104341e-06,
+      "loss": 0.8037,
+      "step": 280
+    },
+    {
+      "epoch": 0.11166730843280709,
+      "grad_norm": 0.2108594924211502,
+      "learning_rate": 9.827876286644698e-06,
+      "loss": 0.8339,
+      "step": 290
+    },
+    {
+      "epoch": 0.11551790527531768,
+      "grad_norm": 0.22630514204502106,
+      "learning_rate": 9.811280460765643e-06,
+      "loss": 0.7914,
+      "step": 300
+    },
+    {
+      "epoch": 0.11936850211782826,
+      "grad_norm": 0.20126600563526154,
+      "learning_rate": 9.793936295756292e-06,
+      "loss": 0.7971,
+      "step": 310
+    },
+    {
+      "epoch": 0.12321909896033885,
+      "grad_norm": 0.24300606548786163,
+      "learning_rate": 9.775846489301279e-06,
+      "loss": 0.8056,
+      "step": 320
+    },
+    {
+      "epoch": 0.12706969580284944,
+      "grad_norm": 0.2103230059146881,
+      "learning_rate": 9.757013855061157e-06,
+      "loss": 0.8091,
+      "step": 330
+    },
+    {
+      "epoch": 0.13092029264536004,
+      "grad_norm": 0.20442284643650055,
+      "learning_rate": 9.737441322234758e-06,
+      "loss": 0.8099,
+      "step": 340
+    },
+    {
+      "epoch": 0.1347708894878706,
+      "grad_norm": 0.24754616618156433,
+      "learning_rate": 9.717131935103605e-06,
+      "loss": 0.8172,
+      "step": 350
+    },
+    {
+      "epoch": 0.1386214863303812,
+      "grad_norm": 0.26217585802078247,
+      "learning_rate": 9.696088852558398e-06,
+      "loss": 0.7965,
+      "step": 360
+    },
+    {
+      "epoch": 0.1424720831728918,
+      "grad_norm": 0.2312014400959015,
+      "learning_rate": 9.674315347607686e-06,
+      "loss": 0.8255,
+      "step": 370
+    },
+    {
+      "epoch": 0.14632268001540238,
+      "grad_norm": 0.2338801771402359,
+      "learning_rate": 9.651814806868796e-06,
+      "loss": 0.7902,
+      "step": 380
+    },
+    {
+      "epoch": 0.15017327685791299,
+      "grad_norm": 0.2136513888835907,
+      "learning_rate": 9.62859073004107e-06,
+      "loss": 0.7938,
+      "step": 390
+    },
+    {
+      "epoch": 0.15402387370042356,
+      "grad_norm": 0.21599580347537994,
+      "learning_rate": 9.604646729361547e-06,
+      "loss": 0.8026,
+      "step": 400
+    },
+    {
+      "epoch": 0.15787447054293416,
+      "grad_norm": 0.2260614037513733,
+      "learning_rate": 9.579986529043099e-06,
+      "loss": 0.799,
+      "step": 410
+    },
+    {
+      "epoch": 0.16172506738544473,
+      "grad_norm": 0.2215307205915451,
+      "learning_rate": 9.554613964695189e-06,
+      "loss": 0.8125,
+      "step": 420
+    },
+    {
+      "epoch": 0.16557566422795533,
+      "grad_norm": 0.2308000922203064,
+      "learning_rate": 9.528532982727281e-06,
+      "loss": 0.7918,
+      "step": 430
+    },
+    {
+      "epoch": 0.16942626107046593,
+      "grad_norm": 0.22776158154010773,
+      "learning_rate": 9.501747639735018e-06,
+      "loss": 0.8022,
+      "step": 440
+    },
+    {
+      "epoch": 0.1732768579129765,
+      "grad_norm": 0.2371487021446228,
+      "learning_rate": 9.474262101869272e-06,
+      "loss": 0.7858,
+      "step": 450
+    },
+    {
+      "epoch": 0.1771274547554871,
+      "grad_norm": 0.24640676379203796,
+      "learning_rate": 9.446080644188141e-06,
+      "loss": 0.795,
+      "step": 460
+    },
+    {
+      "epoch": 0.18097805159799768,
+      "grad_norm": 0.23318250477313995,
+      "learning_rate": 9.417207649992017e-06,
+      "loss": 0.7988,
+      "step": 470
+    },
+    {
+      "epoch": 0.18482864844050828,
+      "grad_norm": 0.24727997183799744,
+      "learning_rate": 9.387647610141816e-06,
+      "loss": 0.7984,
+      "step": 480
+    },
+    {
+      "epoch": 0.18867924528301888,
+      "grad_norm": 0.20307829976081848,
+      "learning_rate": 9.35740512236047e-06,
+      "loss": 0.7902,
+      "step": 490
+    },
+    {
+      "epoch": 0.19252984212552945,
+      "grad_norm": 0.19661548733711243,
+      "learning_rate": 9.326484890517809e-06,
+      "loss": 0.7921,
+      "step": 500
+    },
+    {
+      "epoch": 0.19638043896804006,
+      "grad_norm": 0.23635590076446533,
+      "learning_rate": 9.294891723898926e-06,
+      "loss": 0.7842,
+      "step": 510
+    },
+    {
+      "epoch": 0.20023103581055063,
+      "grad_norm": 0.25400856137275696,
+      "learning_rate": 9.262630536456153e-06,
+      "loss": 0.8085,
+      "step": 520
+    },
+    {
+      "epoch": 0.20408163265306123,
+      "grad_norm": 0.25138556957244873,
+      "learning_rate": 9.229706346044749e-06,
+      "loss": 0.7907,
+      "step": 530
+    },
+    {
+      "epoch": 0.2079322294955718,
+      "grad_norm": 0.23646801710128784,
+      "learning_rate": 9.196124273642422e-06,
+      "loss": 0.8003,
+      "step": 540
+    },
+    {
+      "epoch": 0.2117828263380824,
+      "grad_norm": 0.27683761715888977,
+      "learning_rate": 9.161889542552842e-06,
+      "loss": 0.8045,
+      "step": 550
+    },
+    {
+      "epoch": 0.215633423180593,
+      "grad_norm": 0.21233612298965454,
+      "learning_rate": 9.127007477593193e-06,
+      "loss": 0.8096,
+      "step": 560
+    },
+    {
+      "epoch": 0.21948402002310358,
+      "grad_norm": 0.2502448856830597,
+      "learning_rate": 9.091483504265974e-06,
+      "loss": 0.8269,
+      "step": 570
+    },
+    {
+      "epoch": 0.22333461686561418,
+      "grad_norm": 0.2579082250595093,
+      "learning_rate": 9.055323147915116e-06,
+      "loss": 0.7899,
+      "step": 580
+    },
+    {
+      "epoch": 0.22718521370812475,
+      "grad_norm": 0.2357991486787796,
+      "learning_rate": 9.018532032866585e-06,
+      "loss": 0.8085,
+      "step": 590
+    },
+    {
+      "epoch": 0.23103581055063535,
+      "grad_norm": 0.22110234200954437,
+      "learning_rate": 8.981115881553579e-06,
+      "loss": 0.8063,
+      "step": 600
+    },
+    {
+      "epoch": 0.23488640739314592,
+      "grad_norm": 0.25116410851478577,
+      "learning_rate": 8.943080513626476e-06,
+      "loss": 0.8229,
+      "step": 610
+    },
+    {
+      "epoch": 0.23873700423565652,
+      "grad_norm": 0.2095479667186737,
+      "learning_rate": 8.904431845047641e-06,
+      "loss": 0.8005,
+      "step": 620
+    },
+    {
+      "epoch": 0.24258760107816713,
+      "grad_norm": 0.25354546308517456,
+      "learning_rate": 8.86517588717128e-06,
+      "loss": 0.813,
+      "step": 630
+    },
+    {
+      "epoch": 0.2464381979206777,
+      "grad_norm": 0.21686866879463196,
+      "learning_rate": 8.82531874580844e-06,
+      "loss": 0.8077,
+      "step": 640
+    },
+    {
+      "epoch": 0.25028879476318827,
+      "grad_norm": 0.2210106998682022,
+      "learning_rate": 8.784866620277314e-06,
+      "loss": 0.7699,
+      "step": 650
+    },
+    {
+      "epoch": 0.25413939160569887,
+      "grad_norm": 0.27490314841270447,
+      "learning_rate": 8.743825802439018e-06,
+      "loss": 0.7983,
+      "step": 660
+    },
+    {
+      "epoch": 0.2579899884482095,
+      "grad_norm": 0.22292689979076385,
+      "learning_rate": 8.702202675718956e-06,
+      "loss": 0.8035,
+      "step": 670
+    },
+    {
+      "epoch": 0.2618405852907201,
+      "grad_norm": 0.2531745731830597,
+      "learning_rate": 8.660003714113959e-06,
+      "loss": 0.7882,
+      "step": 680
+    },
+    {
+      "epoch": 0.2656911821332307,
+      "grad_norm": 0.21008700132369995,
+      "learning_rate": 8.617235481185323e-06,
+      "loss": 0.7982,
+      "step": 690
+    },
+    {
+      "epoch": 0.2695417789757412,
+      "grad_norm": 0.22192829847335815,
+      "learning_rate": 8.573904629037931e-06,
+      "loss": 0.8098,
+      "step": 700
+    },
+    {
+      "epoch": 0.2733923758182518,
+      "grad_norm": 0.21810579299926758,
+      "learning_rate": 8.530017897285585e-06,
+      "loss": 0.7893,
+      "step": 710
+    },
+    {
+      "epoch": 0.2772429726607624,
+      "grad_norm": 0.2559935450553894,
+      "learning_rate": 8.485582112002746e-06,
+      "loss": 0.7909,
+      "step": 720
+    },
+    {
+      "epoch": 0.281093569503273,
+      "grad_norm": 0.2243410348892212,
+      "learning_rate": 8.440604184662811e-06,
+      "loss": 0.8022,
+      "step": 730
+    },
+    {
+      "epoch": 0.2849441663457836,
+      "grad_norm": 0.2323644757270813,
+      "learning_rate": 8.395091111063118e-06,
+      "loss": 0.7877,
+      "step": 740
+    },
+    {
+      "epoch": 0.28879476318829417,
+      "grad_norm": 0.2141411304473877,
+      "learning_rate": 8.349049970236822e-06,
+      "loss": 0.783,
+      "step": 750
+    },
+    {
+      "epoch": 0.29264536003080477,
+      "grad_norm": 0.2367885559797287,
+      "learning_rate": 8.302487923351842e-06,
+      "loss": 0.7813,
+      "step": 760
+    },
+    {
+      "epoch": 0.29649595687331537,
+      "grad_norm": 0.21857623755931854,
+      "learning_rate": 8.255412212597021e-06,
+      "loss": 0.7997,
+      "step": 770
+    },
+    {
+      "epoch": 0.30034655371582597,
+      "grad_norm": 0.22147710621356964,
+      "learning_rate": 8.207830160055688e-06,
+      "loss": 0.7936,
+      "step": 780
+    },
+    {
+      "epoch": 0.3041971505583365,
+      "grad_norm": 0.20294037461280823,
+      "learning_rate": 8.159749166566786e-06,
+      "loss": 0.7796,
+      "step": 790
+    },
+    {
+      "epoch": 0.3080477474008471,
+      "grad_norm": 0.2156461775302887,
+      "learning_rate": 8.11117671057377e-06,
+      "loss": 0.8011,
+      "step": 800
+    },
+    {
+      "epoch": 0.3118983442433577,
+      "grad_norm": 0.20514221489429474,
+      "learning_rate": 8.062120346961404e-06,
+      "loss": 0.7999,
+      "step": 810
+    },
+    {
+      "epoch": 0.3157489410858683,
+      "grad_norm": 0.2491036057472229,
+      "learning_rate": 8.012587705880705e-06,
+      "loss": 0.7989,
+      "step": 820
+    },
+    {
+      "epoch": 0.3195995379283789,
+      "grad_norm": 0.2297607809305191,
+      "learning_rate": 7.962586491562146e-06,
+      "loss": 0.7831,
+      "step": 830
+    },
+    {
+      "epoch": 0.32345013477088946,
+      "grad_norm": 0.22131627798080444,
+      "learning_rate": 7.912124481117357e-06,
+      "loss": 0.7819,
+      "step": 840
+    },
+    {
+      "epoch": 0.32730073161340006,
+      "grad_norm": 0.1952383667230606,
+      "learning_rate": 7.861209523329494e-06,
+      "loss": 0.7913,
+      "step": 850
+    },
+    {
+      "epoch": 0.33115132845591067,
+      "grad_norm": 0.21389321982860565,
+      "learning_rate": 7.809849537432432e-06,
+      "loss": 0.7849,
+      "step": 860
+    },
+    {
+      "epoch": 0.33500192529842127,
+      "grad_norm": 0.21146881580352783,
+      "learning_rate": 7.758052511879035e-06,
+      "loss": 0.7876,
+      "step": 870
+    },
+    {
+      "epoch": 0.33885252214093187,
+      "grad_norm": 0.23086167871952057,
+      "learning_rate": 7.705826503098642e-06,
+      "loss": 0.7907,
+      "step": 880
+    },
+    {
+      "epoch": 0.3427031189834424,
+      "grad_norm": 0.22181129455566406,
+      "learning_rate": 7.653179634243979e-06,
+      "loss": 0.7835,
+      "step": 890
+    },
+    {
+      "epoch": 0.346553715825953,
+      "grad_norm": 0.20122453570365906,
+      "learning_rate": 7.600120093927696e-06,
+      "loss": 0.7888,
+      "step": 900
+    },
+    {
+      "epoch": 0.3504043126684636,
+      "grad_norm": 0.23742234706878662,
+      "learning_rate": 7.546656134948729e-06,
+      "loss": 0.7765,
+      "step": 910
+    },
+    {
+      "epoch": 0.3542549095109742,
+      "grad_norm": 0.222365602850914,
+      "learning_rate": 7.492796073008661e-06,
+      "loss": 0.7821,
+      "step": 920
+    },
+    {
+      "epoch": 0.3581055063534848,
+      "grad_norm": 0.2524479031562805,
+      "learning_rate": 7.438548285418326e-06,
+      "loss": 0.7644,
+      "step": 930
+    },
+    {
+      "epoch": 0.36195610319599536,
+      "grad_norm": 0.24065794050693512,
+      "learning_rate": 7.3839212097948e-06,
+      "loss": 0.7949,
+      "step": 940
+    },
+    {
+      "epoch": 0.36580670003850596,
+      "grad_norm": 0.21106627583503723,
+      "learning_rate": 7.328923342749039e-06,
+      "loss": 0.8115,
+      "step": 950
+    },
+    {
+      "epoch": 0.36965729688101656,
+      "grad_norm": 0.20199936628341675,
+      "learning_rate": 7.273563238564319e-06,
+      "loss": 0.8082,
+      "step": 960
+    },
+    {
+      "epoch": 0.37350789372352716,
+      "grad_norm": 0.2530917823314667,
+      "learning_rate": 7.217849507865724e-06,
+      "loss": 0.7786,
+      "step": 970
+    },
+    {
+      "epoch": 0.37735849056603776,
+      "grad_norm": 0.21617966890335083,
+      "learning_rate": 7.161790816280858e-06,
+      "loss": 0.7847,
+      "step": 980
+    },
+    {
+      "epoch": 0.3812090874085483,
+      "grad_norm": 0.26216575503349304,
+      "learning_rate": 7.1053958830920125e-06,
+      "loss": 0.776,
+      "step": 990
+    },
+    {
+      "epoch": 0.3850596842510589,
+      "grad_norm": 0.2227257639169693,
+      "learning_rate": 7.0486734798799715e-06,
+      "loss": 0.784,
+      "step": 1000
+    },
+    {
+      "epoch": 0.3889102810935695,
+      "grad_norm": 0.21508412063121796,
+      "learning_rate": 6.991632429159708e-06,
+      "loss": 0.8034,
+      "step": 1010
+    },
+    {
+      "epoch": 0.3927608779360801,
+      "grad_norm": 0.2289504110813141,
+      "learning_rate": 6.934281603008132e-06,
+      "loss": 0.8042,
+      "step": 1020
+    },
+    {
+      "epoch": 0.39661147477859066,
+      "grad_norm": 0.21158353984355927,
+      "learning_rate": 6.876629921684143e-06,
+      "loss": 0.8003,
+      "step": 1030
+    },
+    {
+      "epoch": 0.40046207162110126,
+      "grad_norm": 0.277248352766037,
+      "learning_rate": 6.818686352241199e-06,
+      "loss": 0.7889,
+      "step": 1040
+    },
+    {
+      "epoch": 0.40431266846361186,
+      "grad_norm": 0.2058485597372055,
+      "learning_rate": 6.760459907132573e-06,
+      "loss": 0.7853,
+      "step": 1050
+    },
+    {
+      "epoch": 0.40816326530612246,
+      "grad_norm": 0.20738479495048523,
+      "learning_rate": 6.701959642809589e-06,
+      "loss": 0.7906,
+      "step": 1060
+    },
+    {
+      "epoch": 0.41201386214863306,
+      "grad_norm": 0.23198120296001434,
+      "learning_rate": 6.643194658312987e-06,
+      "loss": 0.782,
+      "step": 1070
+    },
+    {
+      "epoch": 0.4158644589911436,
+      "grad_norm": 0.20768007636070251,
+      "learning_rate": 6.584174093857676e-06,
+      "loss": 0.8074,
+      "step": 1080
+    },
+    {
+      "epoch": 0.4197150558336542,
+      "grad_norm": 0.2362997978925705,
+      "learning_rate": 6.5249071294110735e-06,
+      "loss": 0.7883,
+      "step": 1090
+    },
+    {
+      "epoch": 0.4235656526761648,
+      "grad_norm": 0.24057388305664062,
+      "learning_rate": 6.465402983265274e-06,
+      "loss": 0.8003,
+      "step": 1100
+    },
+    {
+      "epoch": 0.4274162495186754,
+      "grad_norm": 0.22113044559955597,
+      "learning_rate": 6.405670910603243e-06,
+      "loss": 0.7951,
+      "step": 1110
+    },
+    {
+      "epoch": 0.431266846361186,
+      "grad_norm": 0.25766488909721375,
+      "learning_rate": 6.345720202059291e-06,
+      "loss": 0.7931,
+      "step": 1120
+    },
+    {
+      "epoch": 0.43511744320369655,
+      "grad_norm": 0.22556425631046295,
+      "learning_rate": 6.285560182274008e-06,
+      "loss": 0.7723,
+      "step": 1130
+    },
+    {
+      "epoch": 0.43896804004620715,
+      "grad_norm": 0.22280548512935638,
+      "learning_rate": 6.225200208443936e-06,
+      "loss": 0.7974,
+      "step": 1140
+    },
+    {
+      "epoch": 0.44281863688871775,
+      "grad_norm": 0.21067291498184204,
+      "learning_rate": 6.16464966886616e-06,
+      "loss": 0.7816,
+      "step": 1150
+    },
+    {
+      "epoch": 0.44666923373122835,
+      "grad_norm": 0.24735137820243835,
+      "learning_rate": 6.103917981478065e-06,
+      "loss": 0.8111,
+      "step": 1160
+    },
+    {
+      "epoch": 0.45051983057373896,
+      "grad_norm": 0.2113334685564041,
+      "learning_rate": 6.043014592392489e-06,
+      "loss": 0.8189,
+      "step": 1170
+    },
+    {
+      "epoch": 0.4543704274162495,
+      "grad_norm": 0.20752447843551636,
+      "learning_rate": 5.981948974428485e-06,
+      "loss": 0.7967,
+      "step": 1180
+    },
+    {
+      "epoch": 0.4582210242587601,
+      "grad_norm": 0.2385358363389969,
+      "learning_rate": 5.920730625637934e-06,
+      "loss": 0.7763,
+      "step": 1190
+    },
+    {
+      "epoch": 0.4620716211012707,
+      "grad_norm": 0.22733336687088013,
+      "learning_rate": 5.8593690678282355e-06,
+      "loss": 0.7817,
+      "step": 1200
+    },
+    {
+      "epoch": 0.4659222179437813,
+      "grad_norm": 0.20488208532333374,
+      "learning_rate": 5.797873845081301e-06,
+      "loss": 0.7795,
+      "step": 1210
+    },
+    {
+      "epoch": 0.46977281478629185,
+      "grad_norm": 0.20869646966457367,
+      "learning_rate": 5.736254522269075e-06,
+      "loss": 0.8067,
+      "step": 1220
+    },
+    {
+      "epoch": 0.47362341162880245,
+      "grad_norm": 0.20157702267169952,
+      "learning_rate": 5.6745206835658386e-06,
+      "loss": 0.8001,
+      "step": 1230
+    },
+    {
+      "epoch": 0.47747400847131305,
+      "grad_norm": 0.20262156426906586,
+      "learning_rate": 5.612681930957495e-06,
+      "loss": 0.7788,
+      "step": 1240
+    },
+    {
+      "epoch": 0.48132460531382365,
+      "grad_norm": 0.2189016193151474,
+      "learning_rate": 5.550747882748095e-06,
+      "loss": 0.7738,
+      "step": 1250
+    },
+    {
+      "epoch": 0.48517520215633425,
+      "grad_norm": 0.26426494121551514,
+      "learning_rate": 5.4887281720638195e-06,
+      "loss": 0.7854,
+      "step": 1260
+    },
+    {
+      "epoch": 0.4890257989988448,
+      "grad_norm": 0.23468239605426788,
+      "learning_rate": 5.42663244535466e-06,
+      "loss": 0.7657,
+      "step": 1270
+    },
+    {
+      "epoch": 0.4928763958413554,
+      "grad_norm": 0.21967458724975586,
+      "learning_rate": 5.3644703608940205e-06,
+      "loss": 0.7792,
+      "step": 1280
+    },
+    {
+      "epoch": 0.496726992683866,
+      "grad_norm": 0.2556285560131073,
+      "learning_rate": 5.302251587276491e-06,
+      "loss": 0.7925,
+      "step": 1290
+    },
+    {
+      "epoch": 0.5005775895263765,
+      "grad_norm": 0.23821352422237396,
+      "learning_rate": 5.2399858019140005e-06,
+      "loss": 0.7824,
+      "step": 1300
+    },
+    {
+      "epoch": 0.5044281863688872,
+      "grad_norm": 0.2026471644639969,
+      "learning_rate": 5.177682689530615e-06,
+      "loss": 0.7979,
+      "step": 1310
+    },
+    {
+      "epoch": 0.5082787832113977,
+      "grad_norm": 0.24558252096176147,
+      "learning_rate": 5.11535194065619e-06,
+      "loss": 0.7727,
+      "step": 1320
+    },
+    {
+      "epoch": 0.5121293800539084,
+      "grad_norm": 0.22448225319385529,
+      "learning_rate": 5.053003250119113e-06,
+      "loss": 0.7818,
+      "step": 1330
+    },
+    {
+      "epoch": 0.515979976896419,
+      "grad_norm": 0.21260304749011993,
+      "learning_rate": 4.9906463155383925e-06,
+      "loss": 0.7872,
+      "step": 1340
+    },
+    {
+      "epoch": 0.5198305737389295,
+      "grad_norm": 0.22764699161052704,
+      "learning_rate": 4.9282908358153046e-06,
+      "loss": 0.7823,
+      "step": 1350
+    },
+    {
+      "epoch": 0.5236811705814401,
+      "grad_norm": 0.21872763335704803,
+      "learning_rate": 4.8659465096248345e-06,
+      "loss": 0.8004,
+      "step": 1360
+    },
+    {
+      "epoch": 0.5275317674239507,
+      "grad_norm": 0.23718217015266418,
+      "learning_rate": 4.803623033907169e-06,
+      "loss": 0.7939,
+      "step": 1370
+    },
+    {
+      "epoch": 0.5313823642664613,
+      "grad_norm": 0.2087608277797699,
+      "learning_rate": 4.741330102359441e-06,
+      "loss": 0.8048,
+      "step": 1380
+    },
+    {
+      "epoch": 0.5352329611089719,
+      "grad_norm": 0.2205442488193512,
+      "learning_rate": 4.679077403927996e-06,
+      "loss": 0.7817,
+      "step": 1390
+    },
+    {
+      "epoch": 0.5390835579514824,
+      "grad_norm": 0.24794715642929077,
+      "learning_rate": 4.616874621301373e-06,
+      "loss": 0.7816,
+      "step": 1400
+    },
+    {
+      "epoch": 0.5429341547939931,
+      "grad_norm": 0.22617097198963165,
+      "learning_rate": 4.554731429404293e-06,
+      "loss": 0.7987,
+      "step": 1410
+    },
+    {
+      "epoch": 0.5467847516365036,
+      "grad_norm": 0.23865436017513275,
+      "learning_rate": 4.492657493892822e-06,
+      "loss": 0.7849,
+      "step": 1420
+    },
+    {
+      "epoch": 0.5506353484790143,
+      "grad_norm": 0.227121502161026,
+      "learning_rate": 4.430662469650999e-06,
+      "loss": 0.7971,
+      "step": 1430
+    },
+    {
+      "epoch": 0.5544859453215248,
+      "grad_norm": 0.21364498138427734,
+      "learning_rate": 4.368755999289118e-06,
+      "loss": 0.7955,
+      "step": 1440
+    },
+    {
+      "epoch": 0.5583365421640354,
+      "grad_norm": 0.23610040545463562,
+      "learning_rate": 4.306947711643949e-06,
+      "loss": 0.781,
+      "step": 1450
+    },
+    {
+      "epoch": 0.562187139006546,
+      "grad_norm": 0.22365300357341766,
+      "learning_rate": 4.245247220281068e-06,
+      "loss": 0.785,
+      "step": 1460
+    },
+    {
+      "epoch": 0.5660377358490566,
+      "grad_norm": 0.21084871888160706,
+      "learning_rate": 4.183664121999587e-06,
+      "loss": 0.784,
+      "step": 1470
+    },
+    {
+      "epoch": 0.5698883326915672,
+      "grad_norm": 0.21612055599689484,
+      "learning_rate": 4.122207995339474e-06,
+      "loss": 0.7749,
+      "step": 1480
+    },
+    {
+      "epoch": 0.5737389295340778,
+      "grad_norm": 0.2022586613893509,
+      "learning_rate": 4.060888399091734e-06,
+      "loss": 0.7861,
+      "step": 1490
+    },
+    {
+      "epoch": 0.5775895263765883,
+      "grad_norm": 0.2088070660829544,
+      "learning_rate": 3.999714870811639e-06,
+      "loss": 0.7941,
+      "step": 1500
+    },
+    {
+      "epoch": 0.581440123219099,
+      "grad_norm": 0.20395199954509735,
+      "learning_rate": 3.938696925335275e-06,
+      "loss": 0.7893,
+      "step": 1510
+    },
+    {
+      "epoch": 0.5852907200616095,
+      "grad_norm": 0.2251003086566925,
+      "learning_rate": 3.87784405329962e-06,
+      "loss": 0.7981,
+      "step": 1520
+    },
+    {
+      "epoch": 0.5891413169041202,
+      "grad_norm": 0.2377869039773941,
+      "learning_rate": 3.817165719666382e-06,
+      "loss": 0.7985,
+      "step": 1530
+    },
+    {
+      "epoch": 0.5929919137466307,
+      "grad_norm": 0.22972312569618225,
+      "learning_rate": 3.756671362249841e-06,
+      "loss": 0.7976,
+      "step": 1540
+    },
+    {
+      "epoch": 0.5968425105891413,
+      "grad_norm": 0.23767544329166412,
+      "learning_rate": 3.6963703902488907e-06,
+      "loss": 0.7868,
+      "step": 1550
+    },
+    {
+      "epoch": 0.6006931074316519,
+      "grad_norm": 0.20524664223194122,
+      "learning_rate": 3.6362721827835655e-06,
+      "loss": 0.8013,
+      "step": 1560
+    },
+    {
+      "epoch": 0.6045437042741625,
+      "grad_norm": 0.22464610636234283,
+      "learning_rate": 3.576386087436212e-06,
+      "loss": 0.803,
+      "step": 1570
+    },
+    {
+      "epoch": 0.608394301116673,
+      "grad_norm": 0.2215327024459839,
+      "learning_rate": 3.5167214187975913e-06,
+      "loss": 0.797,
+      "step": 1580
+    },
+    {
+      "epoch": 0.6122448979591837,
+      "grad_norm": 0.22007684409618378,
+      "learning_rate": 3.457287457018086e-06,
+      "loss": 0.7907,
+      "step": 1590
+    },
+    {
+      "epoch": 0.6160954948016942,
+      "grad_norm": 0.21840067207813263,
+      "learning_rate": 3.3980934463643003e-06,
+      "loss": 0.7866,
+      "step": 1600
+    },
+    {
+      "epoch": 0.6199460916442049,
+      "grad_norm": 0.20427870750427246,
+      "learning_rate": 3.3391485937812053e-06,
+      "loss": 0.7654,
+      "step": 1610
+    },
+    {
+      "epoch": 0.6237966884867154,
+      "grad_norm": 0.21983036398887634,
+      "learning_rate": 3.280462067460115e-06,
+      "loss": 0.7667,
+      "step": 1620
+    },
+    {
+      "epoch": 0.627647285329226,
+      "grad_norm": 0.2272370606660843,
+      "learning_rate": 3.222042995412669e-06,
+      "loss": 0.7923,
+      "step": 1630
+    },
+    {
+      "epoch": 0.6314978821717366,
+      "grad_norm": 0.20132261514663696,
+      "learning_rate": 3.163900464051089e-06,
+      "loss": 0.7831,
+      "step": 1640
+    },
+    {
+      "epoch": 0.6353484790142472,
+      "grad_norm": 0.2294384241104126,
+      "learning_rate": 3.1060435167748826e-06,
+      "loss": 0.7969,
+      "step": 1650
+    },
+    {
+      "epoch": 0.6391990758567578,
+      "grad_norm": 0.20586618781089783,
+      "learning_rate": 3.048481152564253e-06,
+      "loss": 0.7724,
+      "step": 1660
+    },
+    {
+      "epoch": 0.6430496726992684,
+      "grad_norm": 0.25002989172935486,
+      "learning_rate": 2.9912223245804017e-06,
+      "loss": 0.7911,
+      "step": 1670
+    },
+    {
+      "epoch": 0.6469002695417789,
+      "grad_norm": 0.23146817088127136,
+      "learning_rate": 2.934275938772981e-06,
+      "loss": 0.7854,
+      "step": 1680
+    },
+    {
+      "epoch": 0.6507508663842896,
+      "grad_norm": 0.21307727694511414,
+      "learning_rate": 2.8776508524948676e-06,
+      "loss": 0.7798,
+      "step": 1690
+    },
+    {
+      "epoch": 0.6546014632268001,
+      "grad_norm": 0.23324774205684662,
+      "learning_rate": 2.821355873124503e-06,
+      "loss": 0.7839,
+      "step": 1700
+    },
+    {
+      "epoch": 0.6584520600693108,
+      "grad_norm": 0.2105952650308609,
+      "learning_rate": 2.7653997566960144e-06,
+      "loss": 0.7862,
+      "step": 1710
+    },
+    {
+      "epoch": 0.6623026569118213,
+      "grad_norm": 0.2491983324289322,
+      "learning_rate": 2.709791206537309e-06,
+      "loss": 0.7796,
+      "step": 1720
+    },
+    {
+      "epoch": 0.6661532537543319,
+      "grad_norm": 0.23735208809375763,
+      "learning_rate": 2.6545388719163747e-06,
+      "loss": 0.7884,
+      "step": 1730
+    },
+    {
+      "epoch": 0.6700038505968425,
+      "grad_norm": 0.24397271871566772,
+      "learning_rate": 2.599651346695979e-06,
+      "loss": 0.7959,
+      "step": 1740
+    },
+    {
+      "epoch": 0.6738544474393531,
+      "grad_norm": 0.19663695991039276,
+      "learning_rate": 2.545137167997007e-06,
+      "loss": 0.7869,
+      "step": 1750
+    },
+    {
+      "epoch": 0.6777050442818637,
+      "grad_norm": 0.20222339034080505,
+      "learning_rate": 2.4910048148705972e-06,
+      "loss": 0.7826,
+      "step": 1760
+    },
+    {
+      "epoch": 0.6815556411243743,
+      "grad_norm": 0.22000083327293396,
+      "learning_rate": 2.4372627069793327e-06,
+      "loss": 0.8157,
+      "step": 1770
+    },
+    {
+      "epoch": 0.6854062379668848,
+      "grad_norm": 0.21383987367153168,
+      "learning_rate": 2.3839192032876486e-06,
+      "loss": 0.7948,
+      "step": 1780
+    },
+    {
+      "epoch": 0.6892568348093955,
+      "grad_norm": 0.21021093428134918,
+      "learning_rate": 2.3309826007617047e-06,
+      "loss": 0.8092,
+      "step": 1790
+    },
+    {
+      "epoch": 0.693107431651906,
+      "grad_norm": 0.2145695984363556,
+      "learning_rate": 2.278461133078881e-06,
+      "loss": 0.776,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6969580284944167,
+      "grad_norm": 0.22775213420391083,
+      "learning_rate": 2.2263629693471258e-06,
+      "loss": 0.7714,
+      "step": 1810
+    },
+    {
+      "epoch": 0.7008086253369272,
+      "grad_norm": 0.2199987918138504,
+      "learning_rate": 2.1746962128343414e-06,
+      "loss": 0.7946,
+      "step": 1820
+    },
+    {
+      "epoch": 0.7046592221794378,
+      "grad_norm": 0.2283315360546112,
+      "learning_rate": 2.1234688997080183e-06,
+      "loss": 0.7772,
+      "step": 1830
+    },
+    {
+      "epoch": 0.7085098190219484,
+      "grad_norm": 0.20141509175300598,
+      "learning_rate": 2.0726889977853053e-06,
+      "loss": 0.7822,
+      "step": 1840
+    },
+    {
+      "epoch": 0.712360415864459,
+      "grad_norm": 0.2355961948633194,
+      "learning_rate": 2.022364405293703e-06,
+      "loss": 0.7967,
+      "step": 1850
+    },
+    {
+      "epoch": 0.7162110127069696,
+      "grad_norm": 0.2220049649477005,
+      "learning_rate": 1.9725029496425814e-06,
+      "loss": 0.7943,
+      "step": 1860
+    },
+    {
+      "epoch": 0.7200616095494802,
+      "grad_norm": 0.2300892174243927,
+      "learning_rate": 1.923112386205727e-06,
+      "loss": 0.7846,
+      "step": 1870
+    },
+    {
+      "epoch": 0.7239122063919907,
+      "grad_norm": 0.20979386568069458,
+      "learning_rate": 1.874200397115079e-06,
+      "loss": 0.7968,
+      "step": 1880
+    },
+    {
+      "epoch": 0.7277628032345014,
+      "grad_norm": 0.19950246810913086,
+      "learning_rate": 1.825774590065857e-06,
+      "loss": 0.7824,
+      "step": 1890
+    },
+    {
+      "epoch": 0.7316134000770119,
+      "grad_norm": 0.21992602944374084,
+      "learning_rate": 1.7778424971332813e-06,
+      "loss": 0.7928,
+      "step": 1900
+    },
+    {
+      "epoch": 0.7354639969195226,
+      "grad_norm": 0.21857032179832458,
+      "learning_rate": 1.7304115736010407e-06,
+      "loss": 0.7931,
+      "step": 1910
+    },
+    {
+      "epoch": 0.7393145937620331,
+      "grad_norm": 0.21143482625484467,
+      "learning_rate": 1.6834891968017108e-06,
+      "loss": 0.7789,
+      "step": 1920
+    },
+    {
+      "epoch": 0.7431651906045437,
+      "grad_norm": 0.22282525897026062,
+      "learning_rate": 1.6370826649692884e-06,
+      "loss": 0.801,
+      "step": 1930
+    },
+    {
+      "epoch": 0.7470157874470543,
+      "grad_norm": 0.21034716069698334,
+      "learning_rate": 1.5911991961040463e-06,
+      "loss": 0.7967,
+      "step": 1940
+    },
+    {
+      "epoch": 0.7508663842895649,
+      "grad_norm": 0.1992741823196411,
+      "learning_rate": 1.545845926849851e-06,
+      "loss": 0.7762,
+      "step": 1950
+    },
+    {
+      "epoch": 0.7547169811320755,
+      "grad_norm": 0.22958669066429138,
+      "learning_rate": 1.5010299113841397e-06,
+      "loss": 0.8059,
+      "step": 1960
+    },
+    {
+      "epoch": 0.7585675779745861,
+      "grad_norm": 0.2208956778049469,
+      "learning_rate": 1.4567581203207238e-06,
+      "loss": 0.7941,
+      "step": 1970
+    },
+    {
+      "epoch": 0.7624181748170966,
+      "grad_norm": 0.22012658417224884,
+      "learning_rate": 1.413037439625593e-06,
+      "loss": 0.7933,
+      "step": 1980
+    },
+    {
+      "epoch": 0.7662687716596073,
+      "grad_norm": 0.2552066147327423,
+      "learning_rate": 1.3698746695458847e-06,
+      "loss": 0.7798,
+      "step": 1990
+    },
+    {
+      "epoch": 0.7701193685021178,
+      "grad_norm": 0.2073410302400589,
+      "learning_rate": 1.3272765235521745e-06,
+      "loss": 0.7828,
+      "step": 2000
+    },
+    {
+      "epoch": 0.7739699653446284,
+      "grad_norm": 0.20402590930461884,
+      "learning_rate": 1.2852496272942877e-06,
+      "loss": 0.7884,
+      "step": 2010
+    },
+    {
+      "epoch": 0.777820562187139,
+      "grad_norm": 0.21308594942092896,
+      "learning_rate": 1.2438005175707396e-06,
+      "loss": 0.7722,
+      "step": 2020
+    },
+    {
+      "epoch": 0.7816711590296496,
+      "grad_norm": 0.22001121938228607,
+      "learning_rate": 1.2029356413120297e-06,
+      "loss": 0.7669,
+      "step": 2030
+    },
+    {
+      "epoch": 0.7855217558721602,
+      "grad_norm": 0.228388711810112,
+      "learning_rate": 1.1626613545778786e-06,
+      "loss": 0.7827,
+      "step": 2040
+    },
+    {
+      "epoch": 0.7893723527146708,
+      "grad_norm": 0.21807453036308289,
+      "learning_rate": 1.1229839215686294e-06,
+      "loss": 0.7851,
+      "step": 2050
+    },
+    {
+      "epoch": 0.7932229495571813,
+      "grad_norm": 0.19019527733325958,
+      "learning_rate": 1.0839095136509188e-06,
+      "loss": 0.7717,
+      "step": 2060
+    },
+    {
+      "epoch": 0.797073546399692,
+      "grad_norm": 0.23125287890434265,
+      "learning_rate": 1.045444208397791e-06,
+      "loss": 0.7883,
+      "step": 2070
+    },
+    {
+      "epoch": 0.8009241432422025,
+      "grad_norm": 0.21540671586990356,
+      "learning_rate": 1.007593988643402e-06,
+      "loss": 0.7635,
+      "step": 2080
+    },
+    {
+      "epoch": 0.8047747400847132,
+      "grad_norm": 0.1919136643409729,
+      "learning_rate": 9.703647415524613e-07,
+      "loss": 0.7901,
+      "step": 2090
+    },
+    {
+      "epoch": 0.8086253369272237,
+      "grad_norm": 0.20898126065731049,
+      "learning_rate": 9.337622577045485e-07,
+      "loss": 0.7851,
+      "step": 2100
+    },
+    {
+      "epoch": 0.8124759337697343,
+      "grad_norm": 0.21763575077056885,
+      "learning_rate": 8.977922301934588e-07,
+      "loss": 0.7838,
+      "step": 2110
+    },
+    {
+      "epoch": 0.8163265306122449,
+      "grad_norm": 0.22243542969226837,
+      "learning_rate": 8.624602537417009e-07,
+      "loss": 0.7982,
+      "step": 2120
+    },
+    {
+      "epoch": 0.8201771274547555,
+      "grad_norm": 0.22607898712158203,
+      "learning_rate": 8.277718238303123e-07,
+      "loss": 0.7803,
+      "step": 2130
+    },
+    {
+      "epoch": 0.8240277242972661,
+      "grad_norm": 0.20837867259979248,
+      "learning_rate": 7.937323358440935e-07,
+      "loss": 0.7825,
+      "step": 2140
+    },
+    {
+      "epoch": 0.8278783211397767,
+      "grad_norm": 0.20559263229370117,
+      "learning_rate": 7.603470842324223e-07,
+      "loss": 0.7732,
+      "step": 2150
+    },
+    {
+      "epoch": 0.8317289179822872,
+      "grad_norm": 0.22006885707378387,
+      "learning_rate": 7.276212616857575e-07,
+      "loss": 0.7936,
+      "step": 2160
+    },
+    {
+      "epoch": 0.8355795148247979,
+      "grad_norm": 0.2071574330329895,
+      "learning_rate": 6.955599583279837e-07,
+      "loss": 0.7879,
+      "step": 2170
+    },
+    {
+      "epoch": 0.8394301116673084,
+      "grad_norm": 0.20808923244476318,
+      "learning_rate": 6.641681609246981e-07,
+      "loss": 0.7813,
+      "step": 2180
+    },
+    {
+      "epoch": 0.8432807085098191,
+      "grad_norm": 0.24438923597335815,
+      "learning_rate": 6.334507521075711e-07,
+      "loss": 0.7853,
+      "step": 2190
+    },
+    {
+      "epoch": 0.8471313053523296,
+      "grad_norm": 0.21383114159107208,
+      "learning_rate": 6.034125096149179e-07,
+      "loss": 0.7858,
+      "step": 2200
+    },
+    {
+      "epoch": 0.8509819021948402,
+      "grad_norm": 0.216257706284523,
+      "learning_rate": 5.740581055485722e-07,
+      "loss": 0.7559,
+      "step": 2210
+    },
+    {
+      "epoch": 0.8548324990373508,
+      "grad_norm": 0.21377217769622803,
+      "learning_rate": 5.453921056471961e-07,
+      "loss": 0.7906,
+      "step": 2220
+    },
+    {
+      "epoch": 0.8586830958798614,
+      "grad_norm": 0.22634518146514893,
+      "learning_rate": 5.174189685761288e-07,
+      "loss": 0.7839,
+      "step": 2230
+    },
+    {
+      "epoch": 0.862533692722372,
+      "grad_norm": 0.1989559829235077,
+      "learning_rate": 4.901430452338979e-07,
+      "loss": 0.7933,
+      "step": 2240
+    },
+    {
+      "epoch": 0.8663842895648826,
+      "grad_norm": 0.20628632605075836,
+      "learning_rate": 4.6356857807548286e-07,
+      "loss": 0.7825,
+      "step": 2250
+    },
+    {
+      "epoch": 0.8702348864073931,
+      "grad_norm": 0.2297450453042984,
+      "learning_rate": 4.376997004524541e-07,
+      "loss": 0.7642,
+      "step": 2260
+    },
+    {
+      "epoch": 0.8740854832499038,
+      "grad_norm": 0.20333047211170197,
+      "learning_rate": 4.12540435970073e-07,
+      "loss": 0.7847,
+      "step": 2270
+    },
+    {
+      "epoch": 0.8779360800924143,
+      "grad_norm": 0.20554882287979126,
+      "learning_rate": 3.880946978614703e-07,
+      "loss": 0.786,
+      "step": 2280
+    },
+    {
+      "epoch": 0.881786676934925,
+      "grad_norm": 0.2073599100112915,
+      "learning_rate": 3.643662883789878e-07,
+      "loss": 0.7948,
+      "step": 2290
+    },
+    {
+      "epoch": 0.8856372737774355,
+      "grad_norm": 0.18859340250492096,
+      "learning_rate": 3.41358898202781e-07,
+      "loss": 0.7736,
+      "step": 2300
+    },
+    {
+      "epoch": 0.889487870619946,
+      "grad_norm": 0.19656231999397278,
+      "learning_rate": 3.190761058667741e-07,
+      "loss": 0.7847,
+      "step": 2310
+    },
+    {
+      "epoch": 0.8933384674624567,
+      "grad_norm": 0.19972918927669525,
+      "learning_rate": 2.9752137720206463e-07,
+      "loss": 0.7889,
+      "step": 2320
+    },
+    {
+      "epoch": 0.8971890643049673,
+      "grad_norm": 0.21642574667930603,
+      "learning_rate": 2.7669806479785067e-07,
+      "loss": 0.797,
+      "step": 2330
+    },
+    {
+      "epoch": 0.9010396611474779,
+      "grad_norm": 0.21202734112739563,
+      "learning_rate": 2.566094074799763e-07,
+      "loss": 0.7909,
+      "step": 2340
+    },
+    {
+      "epoch": 0.9048902579899885,
+      "grad_norm": 0.21446028351783752,
+      "learning_rate": 2.3725852980716557e-07,
+      "loss": 0.7678,
+      "step": 2350
+    },
+    {
+      "epoch": 0.908740854832499,
+      "grad_norm": 0.20001286268234253,
+      "learning_rate": 2.1864844158503983e-07,
+      "loss": 0.7831,
+      "step": 2360
+    },
+    {
+      "epoch": 0.9125914516750097,
+      "grad_norm": 0.23879706859588623,
+      "learning_rate": 2.007820373979702e-07,
+      "loss": 0.7777,
+      "step": 2370
+    },
+    {
+      "epoch": 0.9164420485175202,
+      "grad_norm": 0.20441369712352753,
+      "learning_rate": 1.8366209615886532e-07,
+      "loss": 0.7865,
+      "step": 2380
+    },
+    {
+      "epoch": 0.9202926453600309,
+      "grad_norm": 0.19292804598808289,
+      "learning_rate": 1.6729128067693613e-07,
+      "loss": 0.798,
+      "step": 2390
+    },
+    {
+      "epoch": 0.9241432422025414,
+      "grad_norm": 0.22221052646636963,
+      "learning_rate": 1.5167213724353426e-07,
+      "loss": 0.7924,
+      "step": 2400
+    },
+    {
+      "epoch": 0.927993839045052,
+      "grad_norm": 0.20005159080028534,
+      "learning_rate": 1.3680709523610058e-07,
+      "loss": 0.7706,
+      "step": 2410
+    },
+    {
+      "epoch": 0.9318444358875626,
+      "grad_norm": 0.2114497274160385,
+      "learning_rate": 1.2269846674030418e-07,
+      "loss": 0.8023,
+      "step": 2420
+    },
+    {
+      "epoch": 0.9356950327300732,
+      "grad_norm": 0.20627829432487488,
+      "learning_rate": 1.0934844619042595e-07,
+      "loss": 0.7942,
+      "step": 2430
+    },
+    {
+      "epoch": 0.9395456295725837,
+      "grad_norm": 0.2212829887866974,
+      "learning_rate": 9.675911002803961e-08,
+      "loss": 0.786,
+      "step": 2440
+    },
+    {
+      "epoch": 0.9433962264150944,
+      "grad_norm": 0.23149928450584412,
+      "learning_rate": 8.493241637904381e-08,
+      "loss": 0.7739,
+      "step": 2450
+    },
+    {
+      "epoch": 0.9472468232576049,
+      "grad_norm": 0.22091519832611084,
+      "learning_rate": 7.387020474909689e-08,
+      "loss": 0.8006,
+      "step": 2460
+    },
+    {
+      "epoch": 0.9510974201001156,
+      "grad_norm": 0.21787580847740173,
+      "learning_rate": 6.357419573750856e-08,
+      "loss": 0.7964,
+      "step": 2470
+    },
+    {
+      "epoch": 0.9549480169426261,
+      "grad_norm": 0.2079465687274933,
+      "learning_rate": 5.4045990769613365e-08,
+      "loss": 0.781,
+      "step": 2480
+    },
+    {
+      "epoch": 0.9587986137851366,
+      "grad_norm": 0.20085163414478302,
+      "learning_rate": 4.5287071847694434e-08,
+      "loss": 0.7828,
+      "step": 2490
+    },
+    {
+      "epoch": 0.9626492106276473,
+      "grad_norm": 0.1979106366634369,
+      "learning_rate": 3.7298801320467817e-08,
+      "loss": 0.7837,
+      "step": 2500
+    },
+    {
+      "epoch": 0.9664998074701578,
+      "grad_norm": 0.21052402257919312,
+      "learning_rate": 3.0082421671192576e-08,
+      "loss": 0.782,
+      "step": 2510
+    },
+    {
+      "epoch": 0.9703504043126685,
+      "grad_norm": 0.208291694521904,
+      "learning_rate": 2.3639055324410355e-08,
+      "loss": 0.7958,
+      "step": 2520
+    },
+    {
+      "epoch": 0.974201001155179,
+      "grad_norm": 0.2125210165977478,
+      "learning_rate": 1.7969704471369477e-08,
+      "loss": 0.7782,
+      "step": 2530
+    },
+    {
+      "epoch": 0.9780515979976896,
+      "grad_norm": 0.20064061880111694,
+      "learning_rate": 1.3075250914142413e-08,
+      "loss": 0.7769,
+      "step": 2540
+    },
+    {
+      "epoch": 0.9819021948402002,
+      "grad_norm": 0.19772759079933167,
+      "learning_rate": 8.956455928476049e-09,
+      "loss": 0.7944,
+      "step": 2550
+    },
+    {
+      "epoch": 0.9857527916827108,
+      "grad_norm": 0.20585064589977264,
+      "learning_rate": 5.6139601453786275e-09,
+      "loss": 0.7797,
+      "step": 2560
+    },
+    {
+      "epoch": 0.9896033885252215,
+      "grad_norm": 0.19706140458583832,
+      "learning_rate": 3.0482834514805602e-09,
+      "loss": 0.7746,
+      "step": 2570
+    },
+    {
+      "epoch": 0.993453985367732,
+      "grad_norm": 0.19538451731204987,
+      "learning_rate": 1.2598249081713366e-09,
+      "loss": 0.7666,
+      "step": 2580
+    },
+    {
+      "epoch": 0.9973045822102425,
+      "grad_norm": 0.20087800920009613,
+      "learning_rate": 2.488626895302826e-10,
+      "loss": 0.7964,
+      "step": 2590
+    },
+    {
+      "epoch": 1.0,
+      "step": 2597,
+      "total_flos": 9.658702345655026e+18,
+      "train_loss": 0.7998714770176432,
+      "train_runtime": 15063.7348,
+      "train_samples_per_second": 44.139,
+      "train_steps_per_second": 0.172
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2597,
+  "num_input_tokens_seen": 584317136,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.658702345655026e+18,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

continuous/smollm2/webssl_mae1b_full2b_224/finetune/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60fbdce2033bff23bdc8508a9fbfcf3b2e4704b5404475f732772e0c3925cafc
+size 6264

continuous/smollm2/webssl_mae1b_full2b_224/pretrain/config.json

@@ -0,0 +1,70 @@
+{
+  "add_faster_video": false,
+  "add_time_instruction": false,
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "faster_token_stride": 10,
+  "force_sample": false,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_aspect_ratio": "square",
+  "image_crop_resolution": null,
+  "image_grid_pinpoints": null,
+  "image_split_resolution": null,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mm_hidden_size": 1536,
+  "mm_newline_position": "grid",
+  "mm_patch_merge_type": "flat",
+  "mm_projector_lr": null,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_resampler_type": null,
+  "mm_spatial_pool_mode": "bilinear",
+  "mm_spatial_pool_stride": null,
+  "mm_tunable_parts": "mm_mlp_adapter",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -1,
+  "mm_vision_tower": "hf:/cache/ckpt/download/tokenizer/continuous/webssl/webssl-mae1b-full2b-224",
+  "mm_vision_tower_lr": null,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "pos_skipping_range": 4096,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 130000,
+  "tie_word_embeddings": true,
+  "tokenizer_model_max_length": 8192,
+  "tokenizer_padding_side": "right",
+  "transformers.js_config": {
+    "dtype": "q4",
+    "kv_cache_dtype": {
+      "fp16": "float16",
+      "q4f16": "float16"
+    },
+    "use_external_data_format": {
+      "model.onnx": true,
+      "model_fp16.onnx": true
+    }
+  },
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "use_mm_proj": true,
+  "use_pos_skipping": false,
+  "vision_tower_pretrained": null,
+  "vocab_size": 49152
+}

continuous/smollm2/webssl_mae1b_full2b_224/pretrain/mm_projector.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffbac13c0f2246fe98dcda3374506060bf9f7223e9616d619862195811a045ab
+size 29378616

continuous/smollm2/webssl_mae1b_full2b_224/pretrain/trainer_state.json

@@ -0,0 +1,421 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 545,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01834862385321101,
+      "grad_norm": 14.783483505249023,
+      "learning_rate": 0.00032142857142857147,
+      "loss": 6.6992,
+      "step": 10
+    },
+    {
+      "epoch": 0.03669724770642202,
+      "grad_norm": 0.36506929993629456,
+      "learning_rate": 0.0006785714285714287,
+      "loss": 3.5313,
+      "step": 20
+    },
+    {
+      "epoch": 0.05504587155963303,
+      "grad_norm": 0.0792313888669014,
+      "learning_rate": 0.0009999907688176173,
+      "loss": 2.8815,
+      "step": 30
+    },
+    {
+      "epoch": 0.07339449541284404,
+      "grad_norm": 0.053451914340257645,
+      "learning_rate": 0.0009988834393115766,
+      "loss": 2.794,
+      "step": 40
+    },
+    {
+      "epoch": 0.09174311926605505,
+      "grad_norm": 0.055412206798791885,
+      "learning_rate": 0.0009959345572940772,
+      "loss": 2.7725,
+      "step": 50
+    },
+    {
+      "epoch": 0.11009174311926606,
+      "grad_norm": 0.05407170578837395,
+      "learning_rate": 0.0009911550081155983,
+      "loss": 2.7465,
+      "step": 60
+    },
+    {
+      "epoch": 0.12844036697247707,
+      "grad_norm": 0.09075163304805756,
+      "learning_rate": 0.0009845624347566433,
+      "loss": 2.7299,
+      "step": 70
+    },
+    {
+      "epoch": 0.14678899082568808,
+      "grad_norm": 0.16680413484573364,
+      "learning_rate": 0.0009761811727013547,
+      "loss": 2.7135,
+      "step": 80
+    },
+    {
+      "epoch": 0.1651376146788991,
+      "grad_norm": 0.1040705069899559,
+      "learning_rate": 0.0009660421601067666,
+      "loss": 2.6984,
+      "step": 90
+    },
+    {
+      "epoch": 0.1834862385321101,
+      "grad_norm": 0.13101784884929657,
+      "learning_rate": 0.0009541828235992926,
+      "loss": 2.6917,
+      "step": 100
+    },
+    {
+      "epoch": 0.2018348623853211,
+      "grad_norm": 0.1684999316930771,
+      "learning_rate": 0.0009406469401200151,
+      "loss": 2.6773,
+      "step": 110
+    },
+    {
+      "epoch": 0.22018348623853212,
+      "grad_norm": 0.26930955052375793,
+      "learning_rate": 0.0009254844753287492,
+      "loss": 2.6721,
+      "step": 120
+    },
+    {
+      "epoch": 0.23853211009174313,
+      "grad_norm": 0.32600662112236023,
+      "learning_rate": 0.0009087513991633924,
+      "loss": 2.6616,
+      "step": 130
+    },
+    {
+      "epoch": 0.25688073394495414,
+      "grad_norm": 0.2032058835029602,
+      "learning_rate": 0.0008905094792353917,
+      "loss": 2.6586,
+      "step": 140
+    },
+    {
+      "epoch": 0.27522935779816515,
+      "grad_norm": 0.11723765730857849,
+      "learning_rate": 0.0008708260528239789,
+      "loss": 2.6552,
+      "step": 150
+    },
+    {
+      "epoch": 0.29357798165137616,
+      "grad_norm": 0.23693566024303436,
+      "learning_rate": 0.0008497737783108237,
+      "loss": 2.6417,
+      "step": 160
+    },
+    {
+      "epoch": 0.3119266055045872,
+      "grad_norm": 0.19866740703582764,
+      "learning_rate": 0.0008274303669726426,
+      "loss": 2.6392,
+      "step": 170
+    },
+    {
+      "epoch": 0.3302752293577982,
+      "grad_norm": 0.25327616930007935,
+      "learning_rate": 0.0008038782961218137,
+      "loss": 2.6281,
+      "step": 180
+    },
+    {
+      "epoch": 0.3486238532110092,
+      "grad_norm": 0.1933378428220749,
+      "learning_rate": 0.000779204504653894,
+      "loss": 2.6283,
+      "step": 190
+    },
+    {
+      "epoch": 0.3669724770642202,
+      "grad_norm": 0.27969875931739807,
+      "learning_rate": 0.0007535000721258791,
+      "loss": 2.6036,
+      "step": 200
+    },
+    {
+      "epoch": 0.3853211009174312,
+      "grad_norm": 0.38866615295410156,
+      "learning_rate": 0.00072685988254984,
+      "loss": 2.6059,
+      "step": 210
+    },
+    {
+      "epoch": 0.4036697247706422,
+      "grad_norm": 0.24000269174575806,
+      "learning_rate": 0.0006993822741429906,
+      "loss": 2.6116,
+      "step": 220
+    },
+    {
+      "epoch": 0.42201834862385323,
+      "grad_norm": 0.24945415556430817,
+      "learning_rate": 0.0006711686763270818,
+      "loss": 2.6101,
+      "step": 230
+    },
+    {
+      "epoch": 0.44036697247706424,
+      "grad_norm": 0.20756439864635468,
+      "learning_rate": 0.0006423232353170831,
+      "loss": 2.6005,
+      "step": 240
+    },
+    {
+      "epoch": 0.45871559633027525,
+      "grad_norm": 0.2332235425710678,
+      "learning_rate": 0.0006129524296812335,
+      "loss": 2.5926,
+      "step": 250
+    },
+    {
+      "epoch": 0.47706422018348627,
+      "grad_norm": 0.23302610218524933,
+      "learning_rate": 0.0005831646772915651,
+      "loss": 2.599,
+      "step": 260
+    },
+    {
+      "epoch": 0.4954128440366973,
+      "grad_norm": 0.28516173362731934,
+      "learning_rate": 0.0005530699351157809,
+      "loss": 2.5796,
+      "step": 270
+    },
+    {
+      "epoch": 0.5137614678899083,
+      "grad_norm": 0.27335235476493835,
+      "learning_rate": 0.0005227792933277943,
+      "loss": 2.5884,
+      "step": 280
+    },
+    {
+      "epoch": 0.5321100917431193,
+      "grad_norm": 0.18794499337673187,
+      "learning_rate": 0.0004924045652352092,
+      "loss": 2.5688,
+      "step": 290
+    },
+    {
+      "epoch": 0.5504587155963303,
+      "grad_norm": 0.21959567070007324,
+      "learning_rate": 0.0004620578745374633,
+      "loss": 2.5709,
+      "step": 300
+    },
+    {
+      "epoch": 0.5688073394495413,
+      "grad_norm": 0.24993987381458282,
+      "learning_rate": 0.0004318512414382059,
+      "loss": 2.578,
+      "step": 310
+    },
+    {
+      "epoch": 0.5871559633027523,
+      "grad_norm": 0.29147183895111084,
+      "learning_rate": 0.0004018961691397148,
+      "loss": 2.5778,
+      "step": 320
+    },
+    {
+      "epoch": 0.6055045871559633,
+      "grad_norm": 0.2920821011066437,
+      "learning_rate": 0.0003723032322457458,
+      "loss": 2.5595,
+      "step": 330
+    },
+    {
+      "epoch": 0.6238532110091743,
+      "grad_norm": 0.2623371481895447,
+      "learning_rate": 0.0003431816685921625,
+      "loss": 2.5644,
+      "step": 340
+    },
+    {
+      "epoch": 0.6422018348623854,
+      "grad_norm": 0.24592527747154236,
+      "learning_rate": 0.0003146389760120416,
+      "loss": 2.5693,
+      "step": 350
+    },
+    {
+      "epoch": 0.6605504587155964,
+      "grad_norm": 0.16033832728862762,
+      "learning_rate": 0.00028678051552373485,
+      "loss": 2.5679,
+      "step": 360
+    },
+    {
+      "epoch": 0.6788990825688074,
+      "grad_norm": 0.19608144462108612,
+      "learning_rate": 0.0002597091224066581,
+      "loss": 2.5588,
+      "step": 370
+    },
+    {
+      "epoch": 0.6972477064220184,
+      "grad_norm": 0.23160867393016815,
+      "learning_rate": 0.0002335247266004629,
+      "loss": 2.5686,
+      "step": 380
+    },
+    {
+      "epoch": 0.7155963302752294,
+      "grad_norm": 0.2762646973133087,
+      "learning_rate": 0.000208323983828828,
+      "loss": 2.5555,
+      "step": 390
+    },
+    {
+      "epoch": 0.7339449541284404,
+      "grad_norm": 0.1998247355222702,
+      "learning_rate": 0.0001841999188095224,
+      "loss": 2.555,
+      "step": 400
+    },
+    {
+      "epoch": 0.7522935779816514,
+      "grad_norm": 0.21092970669269562,
+      "learning_rate": 0.00016124158186777676,
+      "loss": 2.5517,
+      "step": 410
+    },
+    {
+      "epoch": 0.7706422018348624,
+      "grad_norm": 0.26234766840934753,
+      "learning_rate": 0.0001395337202205211,
+      "loss": 2.5473,
+      "step": 420
+    },
+    {
+      "epoch": 0.7889908256880734,
+      "grad_norm": 0.22786784172058105,
+      "learning_rate": 0.00011915646514489292,
+      "loss": 2.552,
+      "step": 430
+    },
+    {
+      "epoch": 0.8073394495412844,
+      "grad_norm": 0.19459250569343567,
+      "learning_rate": 0.00010018503618578818,
+      "loss": 2.5549,
+      "step": 440
+    },
+    {
+      "epoch": 0.8256880733944955,
+      "grad_norm": 0.15452030301094055,
+      "learning_rate": 8.268946349432583e-05,
+      "loss": 2.557,
+      "step": 450
+    },
+    {
+      "epoch": 0.8440366972477065,
+      "grad_norm": 0.1768566519021988,
+      "learning_rate": 6.673432932217117e-05,
+      "loss": 2.5444,
+      "step": 460
+    },
+    {
+      "epoch": 0.8623853211009175,
+      "grad_norm": 0.16679634153842926,
+      "learning_rate": 5.2378529625954684e-05,
+      "loss": 2.5368,
+      "step": 470
+    },
+    {
+      "epoch": 0.8807339449541285,
+      "grad_norm": 0.1330423355102539,
+      "learning_rate": 3.9675056661785556e-05,
+      "loss": 2.5417,
+      "step": 480
+    },
+    {
+      "epoch": 0.8990825688073395,
+      "grad_norm": 0.1542026549577713,
+      "learning_rate": 2.8670803372379538e-05,
+      "loss": 2.5478,
+      "step": 490
+    },
+    {
+      "epoch": 0.9174311926605505,
+      "grad_norm": 0.12619264423847198,
+      "learning_rate": 1.9406390288876586e-05,
+      "loss": 2.5521,
+      "step": 500
+    },
+    {
+      "epoch": 0.9357798165137615,
+      "grad_norm": 0.14093606173992157,
+      "learning_rate": 1.191601558631461e-05,
+      "loss": 2.5414,
+      "step": 510
+    },
+    {
+      "epoch": 0.9541284403669725,
+      "grad_norm": 0.13440538942813873,
+      "learning_rate": 6.2273288462559305e-06,
+      "loss": 2.5367,
+      "step": 520
+    },
+    {
+      "epoch": 0.9724770642201835,
+      "grad_norm": 0.12056010961532593,
+      "learning_rate": 2.361328992552314e-06,
+      "loss": 2.5406,
+      "step": 530
+    },
+    {
+      "epoch": 0.9908256880733946,
+      "grad_norm": 0.16605940461158752,
+      "learning_rate": 3.322867770044202e-07,
+      "loss": 2.5433,
+      "step": 540
+    },
+    {
+      "epoch": 1.0,
+      "step": 545,
+      "total_flos": 5.72896096951468e+17,
+      "train_loss": 2.701728978288283,
+      "train_runtime": 1789.8182,
+      "train_samples_per_second": 311.835,
+      "train_steps_per_second": 0.305
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 545,
+  "num_input_tokens_seen": 34658176,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.72896096951468e+17,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

continuous/smollm2/webssl_mae300m_full2b_224/finetune/chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

continuous/smollm2/webssl_mae300m_full2b_224/finetune/config.json

@@ -0,0 +1,70 @@
+{
+  "add_faster_video": false,
+  "add_time_instruction": false,
+  "architectures": [
+    "LlavaLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "faster_token_stride": 10,
+  "force_sample": false,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_aspect_ratio": "square",
+  "image_crop_resolution": null,
+  "image_grid_pinpoints": null,
+  "image_split_resolution": null,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mm_hidden_size": 1024,
+  "mm_newline_position": "grid",
+  "mm_patch_merge_type": "flat",
+  "mm_projector_lr": 0.0001,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_resampler_type": null,
+  "mm_spatial_pool_mode": "bilinear",
+  "mm_spatial_pool_stride": null,
+  "mm_tunable_parts": "mm_mlp_adapter,mm_language_model",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -1,
+  "mm_vision_tower": "hf:/cache/ckpt/download/tokenizer/continuous/webssl/webssl-mae300m-full2b-224",
+  "mm_vision_tower_lr": null,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "pos_skipping_range": 4096,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 130000,
+  "tie_word_embeddings": true,
+  "tokenizer_model_max_length": 8192,
+  "tokenizer_padding_side": "right",
+  "transformers.js_config": {
+    "dtype": "q4",
+    "kv_cache_dtype": {
+      "fp16": "float16",
+      "q4f16": "float16"
+    },
+    "use_external_data_format": {
+      "model.onnx": true,
+      "model_fp16.onnx": true
+    }
+  },
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "use_mm_proj": true,
+  "use_pos_skipping": false,
+  "vision_tower_pretrained": null,
+  "vocab_size": 49152
+}

continuous/smollm2/webssl_mae300m_full2b_224/finetune/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    2
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.57.6"
+}

continuous/smollm2/webssl_mae300m_full2b_224/finetune/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c70bab5ac54c38721357fead3ea072f94647518d810682c7e99809698e4277a8
+size 4258048376

continuous/smollm2/webssl_mae300m_full2b_224/finetune/special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

continuous/smollm2/webssl_mae300m_full2b_224/finetune/tokenizer_config.json

@@ -0,0 +1,155 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

continuous/smollm2/webssl_mae300m_full2b_224/finetune/trainer_state.json

@@ -0,0 +1,1856 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 2597,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.003850596842510589,
+      "grad_norm": 3.128727674484253,
+      "learning_rate": 1.153846153846154e-06,
+      "loss": 1.2774,
+      "step": 10
+    },
+    {
+      "epoch": 0.007701193685021178,
+      "grad_norm": 2.1774511337280273,
+      "learning_rate": 2.435897435897436e-06,
+      "loss": 1.1465,
+      "step": 20
+    },
+    {
+      "epoch": 0.011551790527531768,
+      "grad_norm": 2.1382298469543457,
+      "learning_rate": 3.7179487179487184e-06,
+      "loss": 1.046,
+      "step": 30
+    },
+    {
+      "epoch": 0.015402387370042356,
+      "grad_norm": 1.4231430292129517,
+      "learning_rate": 5e-06,
+      "loss": 1.008,
+      "step": 40
+    },
+    {
+      "epoch": 0.019252984212552945,
+      "grad_norm": 1.3686490058898926,
+      "learning_rate": 6.282051282051282e-06,
+      "loss": 0.9894,
+      "step": 50
+    },
+    {
+      "epoch": 0.023103581055063535,
+      "grad_norm": 0.29779157042503357,
+      "learning_rate": 7.564102564102564e-06,
+      "loss": 0.9153,
+      "step": 60
+    },
+    {
+      "epoch": 0.026954177897574125,
+      "grad_norm": 0.24630999565124512,
+      "learning_rate": 8.846153846153847e-06,
+      "loss": 0.8668,
+      "step": 70
+    },
+    {
+      "epoch": 0.030804774740084712,
+      "grad_norm": 0.24847739934921265,
+      "learning_rate": 9.999996111488723e-06,
+      "loss": 0.8777,
+      "step": 80
+    },
+    {
+      "epoch": 0.0346553715825953,
+      "grad_norm": 0.23466753959655762,
+      "learning_rate": 9.999529497453782e-06,
+      "loss": 0.8616,
+      "step": 90
+    },
+    {
+      "epoch": 0.03850596842510589,
+      "grad_norm": 0.2425977736711502,
+      "learning_rate": 9.998285264324193e-06,
+      "loss": 0.8298,
+      "step": 100
+    },
+    {
+      "epoch": 0.04235656526761648,
+      "grad_norm": 0.20346780121326447,
+      "learning_rate": 9.996263605626052e-06,
+      "loss": 0.8188,
+      "step": 110
+    },
+    {
+      "epoch": 0.04620716211012707,
+      "grad_norm": 0.22380052506923676,
+      "learning_rate": 9.99346483580503e-06,
+      "loss": 0.8374,
+      "step": 120
+    },
+    {
+      "epoch": 0.05005775895263766,
+      "grad_norm": 0.2187991738319397,
+      "learning_rate": 9.989889390177467e-06,
+      "loss": 0.8148,
+      "step": 130
+    },
+    {
+      "epoch": 0.05390835579514825,
+      "grad_norm": 0.2187175750732422,
+      "learning_rate": 9.985537824862647e-06,
+      "loss": 0.8272,
+      "step": 140
+    },
+    {
+      "epoch": 0.05775895263765884,
+      "grad_norm": 0.2292032241821289,
+      "learning_rate": 9.980410816696317e-06,
+      "loss": 0.8053,
+      "step": 150
+    },
+    {
+      "epoch": 0.061609549480169425,
+      "grad_norm": 0.20434226095676422,
+      "learning_rate": 9.97450916312541e-06,
+      "loss": 0.8105,
+      "step": 160
+    },
+    {
+      "epoch": 0.06546014632268002,
+      "grad_norm": 0.1846010535955429,
+      "learning_rate": 9.967833782084004e-06,
+      "loss": 0.8109,
+      "step": 170
+    },
+    {
+      "epoch": 0.0693107431651906,
+      "grad_norm": 0.20623241364955902,
+      "learning_rate": 9.960385711850554e-06,
+      "loss": 0.8437,
+      "step": 180
+    },
+    {
+      "epoch": 0.07316134000770119,
+      "grad_norm": 0.22865167260169983,
+      "learning_rate": 9.952166110886394e-06,
+      "loss": 0.8306,
+      "step": 190
+    },
+    {
+      "epoch": 0.07701193685021178,
+      "grad_norm": 0.20614738762378693,
+      "learning_rate": 9.943176257655567e-06,
+      "loss": 0.8293,
+      "step": 200
+    },
+    {
+      "epoch": 0.08086253369272237,
+      "grad_norm": 0.24040907621383667,
+      "learning_rate": 9.933417550425952e-06,
+      "loss": 0.8154,
+      "step": 210
+    },
+    {
+      "epoch": 0.08471313053523297,
+      "grad_norm": 0.21043162047863007,
+      "learning_rate": 9.9228915070518e-06,
+      "loss": 0.8085,
+      "step": 220
+    },
+    {
+      "epoch": 0.08856372737774355,
+      "grad_norm": 0.19664865732192993,
+      "learning_rate": 9.911599764737632e-06,
+      "loss": 0.8029,
+      "step": 230
+    },
+    {
+      "epoch": 0.09241432422025414,
+      "grad_norm": 0.2090807855129242,
+      "learning_rate": 9.899544079783606e-06,
+      "loss": 0.7967,
+      "step": 240
+    },
+    {
+      "epoch": 0.09626492106276473,
+      "grad_norm": 0.19762232899665833,
+      "learning_rate": 9.886726327312338e-06,
+      "loss": 0.8146,
+      "step": 250
+    },
+    {
+      "epoch": 0.10011551790527531,
+      "grad_norm": 0.20491383969783783,
+      "learning_rate": 9.87314850097724e-06,
+      "loss": 0.8136,
+      "step": 260
+    },
+    {
+      "epoch": 0.1039661147477859,
+      "grad_norm": 0.19891582429409027,
+      "learning_rate": 9.858812712652452e-06,
+      "loss": 0.7954,
+      "step": 270
+    },
+    {
+      "epoch": 0.1078167115902965,
+      "grad_norm": 0.17760974168777466,
+      "learning_rate": 9.843721192104341e-06,
+      "loss": 0.7939,
+      "step": 280
+    },
+    {
+      "epoch": 0.11166730843280709,
+      "grad_norm": 0.20113655924797058,
+      "learning_rate": 9.827876286644698e-06,
+      "loss": 0.8245,
+      "step": 290
+    },
+    {
+      "epoch": 0.11551790527531768,
+      "grad_norm": 0.1814250349998474,
+      "learning_rate": 9.811280460765643e-06,
+      "loss": 0.7827,
+      "step": 300
+    },
+    {
+      "epoch": 0.11936850211782826,
+      "grad_norm": 0.18863148987293243,
+      "learning_rate": 9.793936295756292e-06,
+      "loss": 0.7889,
+      "step": 310
+    },
+    {
+      "epoch": 0.12321909896033885,
+      "grad_norm": 0.18996195495128632,
+      "learning_rate": 9.775846489301279e-06,
+      "loss": 0.7966,
+      "step": 320
+    },
+    {
+      "epoch": 0.12706969580284944,
+      "grad_norm": 0.21181362867355347,
+      "learning_rate": 9.757013855061157e-06,
+      "loss": 0.7999,
+      "step": 330
+    },
+    {
+      "epoch": 0.13092029264536004,
+      "grad_norm": 0.19070370495319366,
+      "learning_rate": 9.737441322234758e-06,
+      "loss": 0.8018,
+      "step": 340
+    },
+    {
+      "epoch": 0.1347708894878706,
+      "grad_norm": 0.21252737939357758,
+      "learning_rate": 9.717131935103605e-06,
+      "loss": 0.8079,
+      "step": 350
+    },
+    {
+      "epoch": 0.1386214863303812,
+      "grad_norm": 0.1901092231273651,
+      "learning_rate": 9.696088852558398e-06,
+      "loss": 0.7877,
+      "step": 360
+    },
+    {
+      "epoch": 0.1424720831728918,
+      "grad_norm": 0.1996941864490509,
+      "learning_rate": 9.674315347607686e-06,
+      "loss": 0.8166,
+      "step": 370
+    },
+    {
+      "epoch": 0.14632268001540238,
+      "grad_norm": 0.17803652584552765,
+      "learning_rate": 9.651814806868796e-06,
+      "loss": 0.7823,
+      "step": 380
+    },
+    {
+      "epoch": 0.15017327685791299,
+      "grad_norm": 0.19303348660469055,
+      "learning_rate": 9.62859073004107e-06,
+      "loss": 0.7856,
+      "step": 390
+    },
+    {
+      "epoch": 0.15402387370042356,
+      "grad_norm": 0.17733243107795715,
+      "learning_rate": 9.604646729361547e-06,
+      "loss": 0.7943,
+      "step": 400
+    },
+    {
+      "epoch": 0.15787447054293416,
+      "grad_norm": 0.19720961153507233,
+      "learning_rate": 9.579986529043099e-06,
+      "loss": 0.7908,
+      "step": 410
+    },
+    {
+      "epoch": 0.16172506738544473,
+      "grad_norm": 0.2046402096748352,
+      "learning_rate": 9.554613964695189e-06,
+      "loss": 0.8047,
+      "step": 420
+    },
+    {
+      "epoch": 0.16557566422795533,
+      "grad_norm": 0.18837183713912964,
+      "learning_rate": 9.528532982727281e-06,
+      "loss": 0.7839,
+      "step": 430
+    },
+    {
+      "epoch": 0.16942626107046593,
+      "grad_norm": 0.20396088063716888,
+      "learning_rate": 9.501747639735018e-06,
+      "loss": 0.7933,
+      "step": 440
+    },
+    {
+      "epoch": 0.1732768579129765,
+      "grad_norm": 0.21917465329170227,
+      "learning_rate": 9.474262101869272e-06,
+      "loss": 0.7772,
+      "step": 450
+    },
+    {
+      "epoch": 0.1771274547554871,
+      "grad_norm": 0.20191459357738495,
+      "learning_rate": 9.446080644188141e-06,
+      "loss": 0.7863,
+      "step": 460
+    },
+    {
+      "epoch": 0.18097805159799768,
+      "grad_norm": 0.19500622153282166,
+      "learning_rate": 9.417207649992017e-06,
+      "loss": 0.79,
+      "step": 470
+    },
+    {
+      "epoch": 0.18482864844050828,
+      "grad_norm": 0.2045755535364151,
+      "learning_rate": 9.387647610141816e-06,
+      "loss": 0.7893,
+      "step": 480
+    },
+    {
+      "epoch": 0.18867924528301888,
+      "grad_norm": 0.18821954727172852,
+      "learning_rate": 9.35740512236047e-06,
+      "loss": 0.7817,
+      "step": 490
+    },
+    {
+      "epoch": 0.19252984212552945,
+      "grad_norm": 0.18700410425662994,
+      "learning_rate": 9.326484890517809e-06,
+      "loss": 0.7846,
+      "step": 500
+    },
+    {
+      "epoch": 0.19638043896804006,
+      "grad_norm": 0.1939058154821396,
+      "learning_rate": 9.294891723898926e-06,
+      "loss": 0.7748,
+      "step": 510
+    },
+    {
+      "epoch": 0.20023103581055063,
+      "grad_norm": 0.19083461165428162,
+      "learning_rate": 9.262630536456153e-06,
+      "loss": 0.7997,
+      "step": 520
+    },
+    {
+      "epoch": 0.20408163265306123,
+      "grad_norm": 0.1943548619747162,
+      "learning_rate": 9.229706346044749e-06,
+      "loss": 0.7832,
+      "step": 530
+    },
+    {
+      "epoch": 0.2079322294955718,
+      "grad_norm": 0.19020406901836395,
+      "learning_rate": 9.196124273642422e-06,
+      "loss": 0.7922,
+      "step": 540
+    },
+    {
+      "epoch": 0.2117828263380824,
+      "grad_norm": 0.20573574304580688,
+      "learning_rate": 9.161889542552842e-06,
+      "loss": 0.7964,
+      "step": 550
+    },
+    {
+      "epoch": 0.215633423180593,
+      "grad_norm": 0.19326013326644897,
+      "learning_rate": 9.127007477593193e-06,
+      "loss": 0.8005,
+      "step": 560
+    },
+    {
+      "epoch": 0.21948402002310358,
+      "grad_norm": 0.209241583943367,
+      "learning_rate": 9.091483504265974e-06,
+      "loss": 0.8175,
+      "step": 570
+    },
+    {
+      "epoch": 0.22333461686561418,
+      "grad_norm": 0.2089364230632782,
+      "learning_rate": 9.055323147915116e-06,
+      "loss": 0.7816,
+      "step": 580
+    },
+    {
+      "epoch": 0.22718521370812475,
+      "grad_norm": 0.18560464680194855,
+      "learning_rate": 9.018532032866585e-06,
+      "loss": 0.7991,
+      "step": 590
+    },
+    {
+      "epoch": 0.23103581055063535,
+      "grad_norm": 0.18953585624694824,
+      "learning_rate": 8.981115881553579e-06,
+      "loss": 0.7967,
+      "step": 600
+    },
+    {
+      "epoch": 0.23488640739314592,
+      "grad_norm": 0.20425675809383392,
+      "learning_rate": 8.943080513626476e-06,
+      "loss": 0.8142,
+      "step": 610
+    },
+    {
+      "epoch": 0.23873700423565652,
+      "grad_norm": 0.17463889718055725,
+      "learning_rate": 8.904431845047641e-06,
+      "loss": 0.7913,
+      "step": 620
+    },
+    {
+      "epoch": 0.24258760107816713,
+      "grad_norm": 0.20559769868850708,
+      "learning_rate": 8.86517588717128e-06,
+      "loss": 0.8046,
+      "step": 630
+    },
+    {
+      "epoch": 0.2464381979206777,
+      "grad_norm": 0.1919933706521988,
+      "learning_rate": 8.82531874580844e-06,
+      "loss": 0.7989,
+      "step": 640
+    },
+    {
+      "epoch": 0.25028879476318827,
+      "grad_norm": 0.18427146971225739,
+      "learning_rate": 8.784866620277314e-06,
+      "loss": 0.7616,
+      "step": 650
+    },
+    {
+      "epoch": 0.25413939160569887,
+      "grad_norm": 0.18977832794189453,
+      "learning_rate": 8.743825802439018e-06,
+      "loss": 0.7895,
+      "step": 660
+    },
+    {
+      "epoch": 0.2579899884482095,
+      "grad_norm": 0.18841570615768433,
+      "learning_rate": 8.702202675718956e-06,
+      "loss": 0.796,
+      "step": 670
+    },
+    {
+      "epoch": 0.2618405852907201,
+      "grad_norm": 0.18293704092502594,
+      "learning_rate": 8.660003714113959e-06,
+      "loss": 0.7792,
+      "step": 680
+    },
+    {
+      "epoch": 0.2656911821332307,
+      "grad_norm": 0.17961958050727844,
+      "learning_rate": 8.617235481185323e-06,
+      "loss": 0.7902,
+      "step": 690
+    },
+    {
+      "epoch": 0.2695417789757412,
+      "grad_norm": 0.22533269226551056,
+      "learning_rate": 8.573904629037931e-06,
+      "loss": 0.8012,
+      "step": 700
+    },
+    {
+      "epoch": 0.2733923758182518,
+      "grad_norm": 0.17423057556152344,
+      "learning_rate": 8.530017897285585e-06,
+      "loss": 0.7806,
+      "step": 710
+    },
+    {
+      "epoch": 0.2772429726607624,
+      "grad_norm": 0.18075725436210632,
+      "learning_rate": 8.485582112002746e-06,
+      "loss": 0.7822,
+      "step": 720
+    },
+    {
+      "epoch": 0.281093569503273,
+      "grad_norm": 0.1863936185836792,
+      "learning_rate": 8.440604184662811e-06,
+      "loss": 0.7935,
+      "step": 730
+    },
+    {
+      "epoch": 0.2849441663457836,
+      "grad_norm": 0.19626832008361816,
+      "learning_rate": 8.395091111063118e-06,
+      "loss": 0.7792,
+      "step": 740
+    },
+    {
+      "epoch": 0.28879476318829417,
+      "grad_norm": 0.17706270515918732,
+      "learning_rate": 8.349049970236822e-06,
+      "loss": 0.7747,
+      "step": 750
+    },
+    {
+      "epoch": 0.29264536003080477,
+      "grad_norm": 0.1800338327884674,
+      "learning_rate": 8.302487923351842e-06,
+      "loss": 0.7722,
+      "step": 760
+    },
+    {
+      "epoch": 0.29649595687331537,
+      "grad_norm": 0.1767960786819458,
+      "learning_rate": 8.255412212597021e-06,
+      "loss": 0.7912,
+      "step": 770
+    },
+    {
+      "epoch": 0.30034655371582597,
+      "grad_norm": 0.19529110193252563,
+      "learning_rate": 8.207830160055688e-06,
+      "loss": 0.7845,
+      "step": 780
+    },
+    {
+      "epoch": 0.3041971505583365,
+      "grad_norm": 0.18637274205684662,
+      "learning_rate": 8.159749166566786e-06,
+      "loss": 0.7712,
+      "step": 790
+    },
+    {
+      "epoch": 0.3080477474008471,
+      "grad_norm": 0.1802118867635727,
+      "learning_rate": 8.11117671057377e-06,
+      "loss": 0.7912,
+      "step": 800
+    },
+    {
+      "epoch": 0.3118983442433577,
+      "grad_norm": 0.16973887383937836,
+      "learning_rate": 8.062120346961404e-06,
+      "loss": 0.7913,
+      "step": 810
+    },
+    {
+      "epoch": 0.3157489410858683,
+      "grad_norm": 0.18224148452281952,
+      "learning_rate": 8.012587705880705e-06,
+      "loss": 0.7885,
+      "step": 820
+    },
+    {
+      "epoch": 0.3195995379283789,
+      "grad_norm": 0.18355964124202728,
+      "learning_rate": 7.962586491562146e-06,
+      "loss": 0.7744,
+      "step": 830
+    },
+    {
+      "epoch": 0.32345013477088946,
+      "grad_norm": 0.17722992599010468,
+      "learning_rate": 7.912124481117357e-06,
+      "loss": 0.7731,
+      "step": 840
+    },
+    {
+      "epoch": 0.32730073161340006,
+      "grad_norm": 0.1744288057088852,
+      "learning_rate": 7.861209523329494e-06,
+      "loss": 0.7821,
+      "step": 850
+    },
+    {
+      "epoch": 0.33115132845591067,
+      "grad_norm": 0.18247643113136292,
+      "learning_rate": 7.809849537432432e-06,
+      "loss": 0.7756,
+      "step": 860
+    },
+    {
+      "epoch": 0.33500192529842127,
+      "grad_norm": 0.1759486049413681,
+      "learning_rate": 7.758052511879035e-06,
+      "loss": 0.7784,
+      "step": 870
+    },
+    {
+      "epoch": 0.33885252214093187,
+      "grad_norm": 0.20224730670452118,
+      "learning_rate": 7.705826503098642e-06,
+      "loss": 0.7819,
+      "step": 880
+    },
+    {
+      "epoch": 0.3427031189834424,
+      "grad_norm": 0.20793311297893524,
+      "learning_rate": 7.653179634243979e-06,
+      "loss": 0.7743,
+      "step": 890
+    },
+    {
+      "epoch": 0.346553715825953,
+      "grad_norm": 0.1824011206626892,
+      "learning_rate": 7.600120093927696e-06,
+      "loss": 0.7789,
+      "step": 900
+    },
+    {
+      "epoch": 0.3504043126684636,
+      "grad_norm": 0.205536350607872,
+      "learning_rate": 7.546656134948729e-06,
+      "loss": 0.7684,
+      "step": 910
+    },
+    {
+      "epoch": 0.3542549095109742,
+      "grad_norm": 0.1729743331670761,
+      "learning_rate": 7.492796073008661e-06,
+      "loss": 0.7732,
+      "step": 920
+    },
+    {
+      "epoch": 0.3581055063534848,
+      "grad_norm": 0.20095275342464447,
+      "learning_rate": 7.438548285418326e-06,
+      "loss": 0.7559,
+      "step": 930
+    },
+    {
+      "epoch": 0.36195610319599536,
+      "grad_norm": 0.18378938734531403,
+      "learning_rate": 7.3839212097948e-06,
+      "loss": 0.7859,
+      "step": 940
+    },
+    {
+      "epoch": 0.36580670003850596,
+      "grad_norm": 0.18926501274108887,
+      "learning_rate": 7.328923342749039e-06,
+      "loss": 0.8031,
+      "step": 950
+    },
+    {
+      "epoch": 0.36965729688101656,
+      "grad_norm": 0.1893647462129593,
+      "learning_rate": 7.273563238564319e-06,
+      "loss": 0.7981,
+      "step": 960
+    },
+    {
+      "epoch": 0.37350789372352716,
+      "grad_norm": 0.1861279010772705,
+      "learning_rate": 7.217849507865724e-06,
+      "loss": 0.769,
+      "step": 970
+    },
+    {
+      "epoch": 0.37735849056603776,
+      "grad_norm": 0.1911628544330597,
+      "learning_rate": 7.161790816280858e-06,
+      "loss": 0.7757,
+      "step": 980
+    },
+    {
+      "epoch": 0.3812090874085483,
+      "grad_norm": 0.18706832826137543,
+      "learning_rate": 7.1053958830920125e-06,
+      "loss": 0.7658,
+      "step": 990
+    },
+    {
+      "epoch": 0.3850596842510589,
+      "grad_norm": 0.19030827283859253,
+      "learning_rate": 7.0486734798799715e-06,
+      "loss": 0.7747,
+      "step": 1000
+    },
+    {
+      "epoch": 0.3889102810935695,
+      "grad_norm": 0.17199969291687012,
+      "learning_rate": 6.991632429159708e-06,
+      "loss": 0.7942,
+      "step": 1010
+    },
+    {
+      "epoch": 0.3927608779360801,
+      "grad_norm": 0.1943858414888382,
+      "learning_rate": 6.934281603008132e-06,
+      "loss": 0.794,
+      "step": 1020
+    },
+    {
+      "epoch": 0.39661147477859066,
+      "grad_norm": 0.17353083193302155,
+      "learning_rate": 6.876629921684143e-06,
+      "loss": 0.7911,
+      "step": 1030
+    },
+    {
+      "epoch": 0.40046207162110126,
+      "grad_norm": 0.20635250210762024,
+      "learning_rate": 6.818686352241199e-06,
+      "loss": 0.7785,
+      "step": 1040
+    },
+    {
+      "epoch": 0.40431266846361186,
+      "grad_norm": 0.17347782850265503,
+      "learning_rate": 6.760459907132573e-06,
+      "loss": 0.7756,
+      "step": 1050
+    },
+    {
+      "epoch": 0.40816326530612246,
+      "grad_norm": 0.16575057804584503,
+      "learning_rate": 6.701959642809589e-06,
+      "loss": 0.7809,
+      "step": 1060
+    },
+    {
+      "epoch": 0.41201386214863306,
+      "grad_norm": 0.17695477604866028,
+      "learning_rate": 6.643194658312987e-06,
+      "loss": 0.7729,
+      "step": 1070
+    },
+    {
+      "epoch": 0.4158644589911436,
+      "grad_norm": 0.16624805331230164,
+      "learning_rate": 6.584174093857676e-06,
+      "loss": 0.7978,
+      "step": 1080
+    },
+    {
+      "epoch": 0.4197150558336542,
+      "grad_norm": 0.18926368653774261,
+      "learning_rate": 6.5249071294110735e-06,
+      "loss": 0.7788,
+      "step": 1090
+    },
+    {
+      "epoch": 0.4235656526761648,
+      "grad_norm": 0.18508568406105042,
+      "learning_rate": 6.465402983265274e-06,
+      "loss": 0.7913,
+      "step": 1100
+    },
+    {
+      "epoch": 0.4274162495186754,
+      "grad_norm": 0.2037215530872345,
+      "learning_rate": 6.405670910603243e-06,
+      "loss": 0.7853,
+      "step": 1110
+    },
+    {
+      "epoch": 0.431266846361186,
+      "grad_norm": 0.21204698085784912,
+      "learning_rate": 6.345720202059291e-06,
+      "loss": 0.7828,
+      "step": 1120
+    },
+    {
+      "epoch": 0.43511744320369655,
+      "grad_norm": 0.19982433319091797,
+      "learning_rate": 6.285560182274008e-06,
+      "loss": 0.763,
+      "step": 1130
+    },
+    {
+      "epoch": 0.43896804004620715,
+      "grad_norm": 0.18800078332424164,
+      "learning_rate": 6.225200208443936e-06,
+      "loss": 0.7873,
+      "step": 1140
+    },
+    {
+      "epoch": 0.44281863688871775,
+      "grad_norm": 0.17706623673439026,
+      "learning_rate": 6.16464966886616e-06,
+      "loss": 0.772,
+      "step": 1150
+    },
+    {
+      "epoch": 0.44666923373122835,
+      "grad_norm": 0.20605814456939697,
+      "learning_rate": 6.103917981478065e-06,
+      "loss": 0.8007,
+      "step": 1160
+    },
+    {
+      "epoch": 0.45051983057373896,
+      "grad_norm": 0.1784144788980484,
+      "learning_rate": 6.043014592392489e-06,
+      "loss": 0.8083,
+      "step": 1170
+    },
+    {
+      "epoch": 0.4543704274162495,
+      "grad_norm": 0.17577457427978516,
+      "learning_rate": 5.981948974428485e-06,
+      "loss": 0.7875,
+      "step": 1180
+    },
+    {
+      "epoch": 0.4582210242587601,
+      "grad_norm": 0.16838666796684265,
+      "learning_rate": 5.920730625637934e-06,
+      "loss": 0.7664,
+      "step": 1190
+    },
+    {
+      "epoch": 0.4620716211012707,
+      "grad_norm": 0.18528403341770172,
+      "learning_rate": 5.8593690678282355e-06,
+      "loss": 0.7718,
+      "step": 1200
+    },
+    {
+      "epoch": 0.4659222179437813,
+      "grad_norm": 0.17632438242435455,
+      "learning_rate": 5.797873845081301e-06,
+      "loss": 0.7704,
+      "step": 1210
+    },
+    {
+      "epoch": 0.46977281478629185,
+      "grad_norm": 0.19609399139881134,
+      "learning_rate": 5.736254522269075e-06,
+      "loss": 0.7975,
+      "step": 1220
+    },
+    {
+      "epoch": 0.47362341162880245,
+      "grad_norm": 0.18067030608654022,
+      "learning_rate": 5.6745206835658386e-06,
+      "loss": 0.7911,
+      "step": 1230
+    },
+    {
+      "epoch": 0.47747400847131305,
+      "grad_norm": 0.17082010209560394,
+      "learning_rate": 5.612681930957495e-06,
+      "loss": 0.7696,
+      "step": 1240
+    },
+    {
+      "epoch": 0.48132460531382365,
+      "grad_norm": 0.18130719661712646,
+      "learning_rate": 5.550747882748095e-06,
+      "loss": 0.7638,
+      "step": 1250
+    },
+    {
+      "epoch": 0.48517520215633425,
+      "grad_norm": 0.19786803424358368,
+      "learning_rate": 5.4887281720638195e-06,
+      "loss": 0.7757,
+      "step": 1260
+    },
+    {
+      "epoch": 0.4890257989988448,
+      "grad_norm": 0.18549050390720367,
+      "learning_rate": 5.42663244535466e-06,
+      "loss": 0.7562,
+      "step": 1270
+    },
+    {
+      "epoch": 0.4928763958413554,
+      "grad_norm": 0.19500261545181274,
+      "learning_rate": 5.3644703608940205e-06,
+      "loss": 0.7698,
+      "step": 1280
+    },
+    {
+      "epoch": 0.496726992683866,
+      "grad_norm": 0.17138580977916718,
+      "learning_rate": 5.302251587276491e-06,
+      "loss": 0.7826,
+      "step": 1290
+    },
+    {
+      "epoch": 0.5005775895263765,
+      "grad_norm": 0.1861606240272522,
+      "learning_rate": 5.2399858019140005e-06,
+      "loss": 0.7727,
+      "step": 1300
+    },
+    {
+      "epoch": 0.5044281863688872,
+      "grad_norm": 0.19097018241882324,
+      "learning_rate": 5.177682689530615e-06,
+      "loss": 0.7896,
+      "step": 1310
+    },
+    {
+      "epoch": 0.5082787832113977,
+      "grad_norm": 0.19442465901374817,
+      "learning_rate": 5.11535194065619e-06,
+      "loss": 0.7631,
+      "step": 1320
+    },
+    {
+      "epoch": 0.5121293800539084,
+      "grad_norm": 0.19067588448524475,
+      "learning_rate": 5.053003250119113e-06,
+      "loss": 0.7726,
+      "step": 1330
+    },
+    {
+      "epoch": 0.515979976896419,
+      "grad_norm": 0.1746632605791092,
+      "learning_rate": 4.9906463155383925e-06,
+      "loss": 0.7766,
+      "step": 1340
+    },
+    {
+      "epoch": 0.5198305737389295,
+      "grad_norm": 0.17626504600048065,
+      "learning_rate": 4.9282908358153046e-06,
+      "loss": 0.7721,
+      "step": 1350
+    },
+    {
+      "epoch": 0.5236811705814401,
+      "grad_norm": 0.19402651488780975,
+      "learning_rate": 4.8659465096248345e-06,
+      "loss": 0.7896,
+      "step": 1360
+    },
+    {
+      "epoch": 0.5275317674239507,
+      "grad_norm": 0.19029565155506134,
+      "learning_rate": 4.803623033907169e-06,
+      "loss": 0.7846,
+      "step": 1370
+    },
+    {
+      "epoch": 0.5313823642664613,
+      "grad_norm": 0.17127524316310883,
+      "learning_rate": 4.741330102359441e-06,
+      "loss": 0.7955,
+      "step": 1380
+    },
+    {
+      "epoch": 0.5352329611089719,
+      "grad_norm": 0.18341438472270966,
+      "learning_rate": 4.679077403927996e-06,
+      "loss": 0.7723,
+      "step": 1390
+    },
+    {
+      "epoch": 0.5390835579514824,
+      "grad_norm": 0.18673890829086304,
+      "learning_rate": 4.616874621301373e-06,
+      "loss": 0.772,
+      "step": 1400
+    },
+    {
+      "epoch": 0.5429341547939931,
+      "grad_norm": 0.19303704798221588,
+      "learning_rate": 4.554731429404293e-06,
+      "loss": 0.7886,
+      "step": 1410
+    },
+    {
+      "epoch": 0.5467847516365036,
+      "grad_norm": 0.19691438972949982,
+      "learning_rate": 4.492657493892822e-06,
+      "loss": 0.7751,
+      "step": 1420
+    },
+    {
+      "epoch": 0.5506353484790143,
+      "grad_norm": 0.18694958090782166,
+      "learning_rate": 4.430662469650999e-06,
+      "loss": 0.7877,
+      "step": 1430
+    },
+    {
+      "epoch": 0.5544859453215248,
+      "grad_norm": 0.19545695185661316,
+      "learning_rate": 4.368755999289118e-06,
+      "loss": 0.7858,
+      "step": 1440
+    },
+    {
+      "epoch": 0.5583365421640354,
+      "grad_norm": 0.18604086339473724,
+      "learning_rate": 4.306947711643949e-06,
+      "loss": 0.772,
+      "step": 1450
+    },
+    {
+      "epoch": 0.562187139006546,
+      "grad_norm": 0.18955577909946442,
+      "learning_rate": 4.245247220281068e-06,
+      "loss": 0.776,
+      "step": 1460
+    },
+    {
+      "epoch": 0.5660377358490566,
+      "grad_norm": 0.1806289255619049,
+      "learning_rate": 4.183664121999587e-06,
+      "loss": 0.7738,
+      "step": 1470
+    },
+    {
+      "epoch": 0.5698883326915672,
+      "grad_norm": 0.19797416031360626,
+      "learning_rate": 4.122207995339474e-06,
+      "loss": 0.7656,
+      "step": 1480
+    },
+    {
+      "epoch": 0.5737389295340778,
+      "grad_norm": 0.18174177408218384,
+      "learning_rate": 4.060888399091734e-06,
+      "loss": 0.7757,
+      "step": 1490
+    },
+    {
+      "epoch": 0.5775895263765883,
+      "grad_norm": 0.18349431455135345,
+      "learning_rate": 3.999714870811639e-06,
+      "loss": 0.7855,
+      "step": 1500
+    },
+    {
+      "epoch": 0.581440123219099,
+      "grad_norm": 0.1821211278438568,
+      "learning_rate": 3.938696925335275e-06,
+      "loss": 0.7802,
+      "step": 1510
+    },
+    {
+      "epoch": 0.5852907200616095,
+      "grad_norm": 0.18988126516342163,
+      "learning_rate": 3.87784405329962e-06,
+      "loss": 0.7878,
+      "step": 1520
+    },
+    {
+      "epoch": 0.5891413169041202,
+      "grad_norm": 0.17441615462303162,
+      "learning_rate": 3.817165719666382e-06,
+      "loss": 0.789,
+      "step": 1530
+    },
+    {
+      "epoch": 0.5929919137466307,
+      "grad_norm": 0.1810256987810135,
+      "learning_rate": 3.756671362249841e-06,
+      "loss": 0.7879,
+      "step": 1540
+    },
+    {
+      "epoch": 0.5968425105891413,
+      "grad_norm": 0.17734751105308533,
+      "learning_rate": 3.6963703902488907e-06,
+      "loss": 0.7766,
+      "step": 1550
+    },
+    {
+      "epoch": 0.6006931074316519,
+      "grad_norm": 0.17393942177295685,
+      "learning_rate": 3.6362721827835655e-06,
+      "loss": 0.7917,
+      "step": 1560
+    },
+    {
+      "epoch": 0.6045437042741625,
+      "grad_norm": 0.19008712470531464,
+      "learning_rate": 3.576386087436212e-06,
+      "loss": 0.7927,
+      "step": 1570
+    },
+    {
+      "epoch": 0.608394301116673,
+      "grad_norm": 0.19863112270832062,
+      "learning_rate": 3.5167214187975913e-06,
+      "loss": 0.7859,
+      "step": 1580
+    },
+    {
+      "epoch": 0.6122448979591837,
+      "grad_norm": 0.1644529551267624,
+      "learning_rate": 3.457287457018086e-06,
+      "loss": 0.7808,
+      "step": 1590
+    },
+    {
+      "epoch": 0.6160954948016942,
+      "grad_norm": 0.179550901055336,
+      "learning_rate": 3.3980934463643003e-06,
+      "loss": 0.7769,
+      "step": 1600
+    },
+    {
+      "epoch": 0.6199460916442049,
+      "grad_norm": 0.16358326375484467,
+      "learning_rate": 3.3391485937812053e-06,
+      "loss": 0.7556,
+      "step": 1610
+    },
+    {
+      "epoch": 0.6237966884867154,
+      "grad_norm": 0.18471655249595642,
+      "learning_rate": 3.280462067460115e-06,
+      "loss": 0.7569,
+      "step": 1620
+    },
+    {
+      "epoch": 0.627647285329226,
+      "grad_norm": 0.18375512957572937,
+      "learning_rate": 3.222042995412669e-06,
+      "loss": 0.7814,
+      "step": 1630
+    },
+    {
+      "epoch": 0.6314978821717366,
+      "grad_norm": 0.16955214738845825,
+      "learning_rate": 3.163900464051089e-06,
+      "loss": 0.7729,
+      "step": 1640
+    },
+    {
+      "epoch": 0.6353484790142472,
+      "grad_norm": 0.18556813895702362,
+      "learning_rate": 3.1060435167748826e-06,
+      "loss": 0.787,
+      "step": 1650
+    },
+    {
+      "epoch": 0.6391990758567578,
+      "grad_norm": 0.18260197341442108,
+      "learning_rate": 3.048481152564253e-06,
+      "loss": 0.7627,
+      "step": 1660
+    },
+    {
+      "epoch": 0.6430496726992684,
+      "grad_norm": 0.1939888745546341,
+      "learning_rate": 2.9912223245804017e-06,
+      "loss": 0.7806,
+      "step": 1670
+    },
+    {
+      "epoch": 0.6469002695417789,
+      "grad_norm": 0.19130362570285797,
+      "learning_rate": 2.934275938772981e-06,
+      "loss": 0.7754,
+      "step": 1680
+    },
+    {
+      "epoch": 0.6507508663842896,
+      "grad_norm": 0.1940569132566452,
+      "learning_rate": 2.8776508524948676e-06,
+      "loss": 0.7699,
+      "step": 1690
+    },
+    {
+      "epoch": 0.6546014632268001,
+      "grad_norm": 0.17806202173233032,
+      "learning_rate": 2.821355873124503e-06,
+      "loss": 0.7741,
+      "step": 1700
+    },
+    {
+      "epoch": 0.6584520600693108,
+      "grad_norm": 0.18705031275749207,
+      "learning_rate": 2.7653997566960144e-06,
+      "loss": 0.7768,
+      "step": 1710
+    },
+    {
+      "epoch": 0.6623026569118213,
+      "grad_norm": 0.19861474633216858,
+      "learning_rate": 2.709791206537309e-06,
+      "loss": 0.77,
+      "step": 1720
+    },
+    {
+      "epoch": 0.6661532537543319,
+      "grad_norm": 0.19107548892498016,
+      "learning_rate": 2.6545388719163747e-06,
+      "loss": 0.7789,
+      "step": 1730
+    },
+    {
+      "epoch": 0.6700038505968425,
+      "grad_norm": 0.18329331278800964,
+      "learning_rate": 2.599651346695979e-06,
+      "loss": 0.7857,
+      "step": 1740
+    },
+    {
+      "epoch": 0.6738544474393531,
+      "grad_norm": 0.18248412013053894,
+      "learning_rate": 2.545137167997007e-06,
+      "loss": 0.7773,
+      "step": 1750
+    },
+    {
+      "epoch": 0.6777050442818637,
+      "grad_norm": 0.1791553646326065,
+      "learning_rate": 2.4910048148705972e-06,
+      "loss": 0.7727,
+      "step": 1760
+    },
+    {
+      "epoch": 0.6815556411243743,
+      "grad_norm": 0.18868710100650787,
+      "learning_rate": 2.4372627069793327e-06,
+      "loss": 0.806,
+      "step": 1770
+    },
+    {
+      "epoch": 0.6854062379668848,
+      "grad_norm": 0.17435133457183838,
+      "learning_rate": 2.3839192032876486e-06,
+      "loss": 0.7845,
+      "step": 1780
+    },
+    {
+      "epoch": 0.6892568348093955,
+      "grad_norm": 0.19100575149059296,
+      "learning_rate": 2.3309826007617047e-06,
+      "loss": 0.7993,
+      "step": 1790
+    },
+    {
+      "epoch": 0.693107431651906,
+      "grad_norm": 0.17713719606399536,
+      "learning_rate": 2.278461133078881e-06,
+      "loss": 0.7672,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6969580284944167,
+      "grad_norm": 0.17850613594055176,
+      "learning_rate": 2.2263629693471258e-06,
+      "loss": 0.7613,
+      "step": 1810
+    },
+    {
+      "epoch": 0.7008086253369272,
+      "grad_norm": 0.16899481415748596,
+      "learning_rate": 2.1746962128343414e-06,
+      "loss": 0.7852,
+      "step": 1820
+    },
+    {
+      "epoch": 0.7046592221794378,
+      "grad_norm": 0.1790832281112671,
+      "learning_rate": 2.1234688997080183e-06,
+      "loss": 0.7674,
+      "step": 1830
+    },
+    {
+      "epoch": 0.7085098190219484,
+      "grad_norm": 0.18167425692081451,
+      "learning_rate": 2.0726889977853053e-06,
+      "loss": 0.7717,
+      "step": 1840
+    },
+    {
+      "epoch": 0.712360415864459,
+      "grad_norm": 0.18510675430297852,
+      "learning_rate": 2.022364405293703e-06,
+      "loss": 0.7864,
+      "step": 1850
+    },
+    {
+      "epoch": 0.7162110127069696,
+      "grad_norm": 0.1745166927576065,
+      "learning_rate": 1.9725029496425814e-06,
+      "loss": 0.7847,
+      "step": 1860
+    },
+    {
+      "epoch": 0.7200616095494802,
+      "grad_norm": 0.17960487306118011,
+      "learning_rate": 1.923112386205727e-06,
+      "loss": 0.7751,
+      "step": 1870
+    },
+    {
+      "epoch": 0.7239122063919907,
+      "grad_norm": 0.1867515742778778,
+      "learning_rate": 1.874200397115079e-06,
+      "loss": 0.7862,
+      "step": 1880
+    },
+    {
+      "epoch": 0.7277628032345014,
+      "grad_norm": 0.18598763644695282,
+      "learning_rate": 1.825774590065857e-06,
+      "loss": 0.7726,
+      "step": 1890
+    },
+    {
+      "epoch": 0.7316134000770119,
+      "grad_norm": 0.1978961080312729,
+      "learning_rate": 1.7778424971332813e-06,
+      "loss": 0.783,
+      "step": 1900
+    },
+    {
+      "epoch": 0.7354639969195226,
+      "grad_norm": 0.17468488216400146,
+      "learning_rate": 1.7304115736010407e-06,
+      "loss": 0.7829,
+      "step": 1910
+    },
+    {
+      "epoch": 0.7393145937620331,
+      "grad_norm": 0.1770145297050476,
+      "learning_rate": 1.6834891968017108e-06,
+      "loss": 0.7687,
+      "step": 1920
+    },
+    {
+      "epoch": 0.7431651906045437,
+      "grad_norm": 0.18894433975219727,
+      "learning_rate": 1.6370826649692884e-06,
+      "loss": 0.7906,
+      "step": 1930
+    },
+    {
+      "epoch": 0.7470157874470543,
+      "grad_norm": 0.17252446711063385,
+      "learning_rate": 1.5911991961040463e-06,
+      "loss": 0.7862,
+      "step": 1940
+    },
+    {
+      "epoch": 0.7508663842895649,
+      "grad_norm": 0.1817261427640915,
+      "learning_rate": 1.545845926849851e-06,
+      "loss": 0.767,
+      "step": 1950
+    },
+    {
+      "epoch": 0.7547169811320755,
+      "grad_norm": 0.19731013476848602,
+      "learning_rate": 1.5010299113841397e-06,
+      "loss": 0.7955,
+      "step": 1960
+    },
+    {
+      "epoch": 0.7585675779745861,
+      "grad_norm": 0.18587271869182587,
+      "learning_rate": 1.4567581203207238e-06,
+      "loss": 0.7837,
+      "step": 1970
+    },
+    {
+      "epoch": 0.7624181748170966,
+      "grad_norm": 0.19312596321105957,
+      "learning_rate": 1.413037439625593e-06,
+      "loss": 0.7835,
+      "step": 1980
+    },
+    {
+      "epoch": 0.7662687716596073,
+      "grad_norm": 0.19648084044456482,
+      "learning_rate": 1.3698746695458847e-06,
+      "loss": 0.7701,
+      "step": 1990
+    },
+    {
+      "epoch": 0.7701193685021178,
+      "grad_norm": 0.17600245773792267,
+      "learning_rate": 1.3272765235521745e-06,
+      "loss": 0.7738,
+      "step": 2000
+    },
+    {
+      "epoch": 0.7739699653446284,
+      "grad_norm": 0.17839303612709045,
+      "learning_rate": 1.2852496272942877e-06,
+      "loss": 0.778,
+      "step": 2010
+    },
+    {
+      "epoch": 0.777820562187139,
+      "grad_norm": 0.17329251766204834,
+      "learning_rate": 1.2438005175707396e-06,
+      "loss": 0.7616,
+      "step": 2020
+    },
+    {
+      "epoch": 0.7816711590296496,
+      "grad_norm": 0.1811053454875946,
+      "learning_rate": 1.2029356413120297e-06,
+      "loss": 0.7578,
+      "step": 2030
+    },
+    {
+      "epoch": 0.7855217558721602,
+      "grad_norm": 0.1845693588256836,
+      "learning_rate": 1.1626613545778786e-06,
+      "loss": 0.7725,
+      "step": 2040
+    },
+    {
+      "epoch": 0.7893723527146708,
+      "grad_norm": 0.169509619474411,
+      "learning_rate": 1.1229839215686294e-06,
+      "loss": 0.7746,
+      "step": 2050
+    },
+    {
+      "epoch": 0.7932229495571813,
+      "grad_norm": 0.161871999502182,
+      "learning_rate": 1.0839095136509188e-06,
+      "loss": 0.7615,
+      "step": 2060
+    },
+    {
+      "epoch": 0.797073546399692,
+      "grad_norm": 0.19260168075561523,
+      "learning_rate": 1.045444208397791e-06,
+      "loss": 0.7786,
+      "step": 2070
+    },
+    {
+      "epoch": 0.8009241432422025,
+      "grad_norm": 0.17263107001781464,
+      "learning_rate": 1.007593988643402e-06,
+      "loss": 0.7535,
+      "step": 2080
+    },
+    {
+      "epoch": 0.8047747400847132,
+      "grad_norm": 0.1732078343629837,
+      "learning_rate": 9.703647415524613e-07,
+      "loss": 0.78,
+      "step": 2090
+    },
+    {
+      "epoch": 0.8086253369272237,
+      "grad_norm": 0.18523064255714417,
+      "learning_rate": 9.337622577045485e-07,
+      "loss": 0.7743,
+      "step": 2100
+    },
+    {
+      "epoch": 0.8124759337697343,
+      "grad_norm": 0.18356288969516754,
+      "learning_rate": 8.977922301934588e-07,
+      "loss": 0.7736,
+      "step": 2110
+    },
+    {
+      "epoch": 0.8163265306122449,
+      "grad_norm": 0.18895062804222107,
+      "learning_rate": 8.624602537417009e-07,
+      "loss": 0.7883,
+      "step": 2120
+    },
+    {
+      "epoch": 0.8201771274547555,
+      "grad_norm": 0.18422269821166992,
+      "learning_rate": 8.277718238303123e-07,
+      "loss": 0.7702,
+      "step": 2130
+    },
+    {
+      "epoch": 0.8240277242972661,
+      "grad_norm": 0.18109896779060364,
+      "learning_rate": 7.937323358440935e-07,
+      "loss": 0.7733,
+      "step": 2140
+    },
+    {
+      "epoch": 0.8278783211397767,
+      "grad_norm": 0.18884465098381042,
+      "learning_rate": 7.603470842324223e-07,
+      "loss": 0.7635,
+      "step": 2150
+    },
+    {
+      "epoch": 0.8317289179822872,
+      "grad_norm": 0.1812513768672943,
+      "learning_rate": 7.276212616857575e-07,
+      "loss": 0.7832,
+      "step": 2160
+    },
+    {
+      "epoch": 0.8355795148247979,
+      "grad_norm": 0.17968963086605072,
+      "learning_rate": 6.955599583279837e-07,
+      "loss": 0.7776,
+      "step": 2170
+    },
+    {
+      "epoch": 0.8394301116673084,
+      "grad_norm": 0.17212115228176117,
+      "learning_rate": 6.641681609246981e-07,
+      "loss": 0.7714,
+      "step": 2180
+    },
+    {
+      "epoch": 0.8432807085098191,
+      "grad_norm": 0.18843069672584534,
+      "learning_rate": 6.334507521075711e-07,
+      "loss": 0.7754,
+      "step": 2190
+    },
+    {
+      "epoch": 0.8471313053523296,
+      "grad_norm": 0.18496264517307281,
+      "learning_rate": 6.034125096149179e-07,
+      "loss": 0.7757,
+      "step": 2200
+    },
+    {
+      "epoch": 0.8509819021948402,
+      "grad_norm": 0.1700846254825592,
+      "learning_rate": 5.740581055485722e-07,
+      "loss": 0.7466,
+      "step": 2210
+    },
+    {
+      "epoch": 0.8548324990373508,
+      "grad_norm": 0.16911686956882477,
+      "learning_rate": 5.453921056471961e-07,
+      "loss": 0.7802,
+      "step": 2220
+    },
+    {
+      "epoch": 0.8586830958798614,
+      "grad_norm": 0.1853056102991104,
+      "learning_rate": 5.174189685761288e-07,
+      "loss": 0.7729,
+      "step": 2230
+    },
+    {
+      "epoch": 0.862533692722372,
+      "grad_norm": 0.17809908092021942,
+      "learning_rate": 4.901430452338979e-07,
+      "loss": 0.7835,
+      "step": 2240
+    },
+    {
+      "epoch": 0.8663842895648826,
+      "grad_norm": 0.1777345985174179,
+      "learning_rate": 4.6356857807548286e-07,
+      "loss": 0.7734,
+      "step": 2250
+    },
+    {
+      "epoch": 0.8702348864073931,
+      "grad_norm": 0.20798729360103607,
+      "learning_rate": 4.376997004524541e-07,
+      "loss": 0.7549,
+      "step": 2260
+    },
+    {
+      "epoch": 0.8740854832499038,
+      "grad_norm": 0.1774686872959137,
+      "learning_rate": 4.12540435970073e-07,
+      "loss": 0.7744,
+      "step": 2270
+    },
+    {
+      "epoch": 0.8779360800924143,
+      "grad_norm": 0.17248210310935974,
+      "learning_rate": 3.880946978614703e-07,
+      "loss": 0.7764,
+      "step": 2280
+    },
+    {
+      "epoch": 0.881786676934925,
+      "grad_norm": 0.1740095615386963,
+      "learning_rate": 3.643662883789878e-07,
+      "loss": 0.784,
+      "step": 2290
+    },
+    {
+      "epoch": 0.8856372737774355,
+      "grad_norm": 0.1686910092830658,
+      "learning_rate": 3.41358898202781e-07,
+      "loss": 0.7642,
+      "step": 2300
+    },
+    {
+      "epoch": 0.889487870619946,
+      "grad_norm": 0.17033128440380096,
+      "learning_rate": 3.190761058667741e-07,
+      "loss": 0.7747,
+      "step": 2310
+    },
+    {
+      "epoch": 0.8933384674624567,
+      "grad_norm": 0.16868343949317932,
+      "learning_rate": 2.9752137720206463e-07,
+      "loss": 0.7784,
+      "step": 2320
+    },
+    {
+      "epoch": 0.8971890643049673,
+      "grad_norm": 0.18472805619239807,
+      "learning_rate": 2.7669806479785067e-07,
+      "loss": 0.7875,
+      "step": 2330
+    },
+    {
+      "epoch": 0.9010396611474779,
+      "grad_norm": 0.17140242457389832,
+      "learning_rate": 2.566094074799763e-07,
+      "loss": 0.7808,
+      "step": 2340
+    },
+    {
+      "epoch": 0.9048902579899885,
+      "grad_norm": 0.1776757538318634,
+      "learning_rate": 2.3725852980716557e-07,
+      "loss": 0.7581,
+      "step": 2350
+    },
+    {
+      "epoch": 0.908740854832499,
+      "grad_norm": 0.17488405108451843,
+      "learning_rate": 2.1864844158503983e-07,
+      "loss": 0.774,
+      "step": 2360
+    },
+    {
+      "epoch": 0.9125914516750097,
+      "grad_norm": 0.19718705117702484,
+      "learning_rate": 2.007820373979702e-07,
+      "loss": 0.7672,
+      "step": 2370
+    },
+    {
+      "epoch": 0.9164420485175202,
+      "grad_norm": 0.1798376441001892,
+      "learning_rate": 1.8366209615886532e-07,
+      "loss": 0.777,
+      "step": 2380
+    },
+    {
+      "epoch": 0.9202926453600309,
+      "grad_norm": 0.17529186606407166,
+      "learning_rate": 1.6729128067693613e-07,
+      "loss": 0.7874,
+      "step": 2390
+    },
+    {
+      "epoch": 0.9241432422025414,
+      "grad_norm": 0.19441480934619904,
+      "learning_rate": 1.5167213724353426e-07,
+      "loss": 0.782,
+      "step": 2400
+    },
+    {
+      "epoch": 0.927993839045052,
+      "grad_norm": 0.17525674402713776,
+      "learning_rate": 1.3680709523610058e-07,
+      "loss": 0.7604,
+      "step": 2410
+    },
+    {
+      "epoch": 0.9318444358875626,
+      "grad_norm": 0.17389672994613647,
+      "learning_rate": 1.2269846674030418e-07,
+      "loss": 0.7916,
+      "step": 2420
+    },
+    {
+      "epoch": 0.9356950327300732,
+      "grad_norm": 0.17868967354297638,
+      "learning_rate": 1.0934844619042595e-07,
+      "loss": 0.7841,
+      "step": 2430
+    },
+    {
+      "epoch": 0.9395456295725837,
+      "grad_norm": 0.17634522914886475,
+      "learning_rate": 9.675911002803961e-08,
+      "loss": 0.7755,
+      "step": 2440
+    },
+    {
+      "epoch": 0.9433962264150944,
+      "grad_norm": 0.1853901594877243,
+      "learning_rate": 8.493241637904381e-08,
+      "loss": 0.764,
+      "step": 2450
+    },
+    {
+      "epoch": 0.9472468232576049,
+      "grad_norm": 0.1988186240196228,
+      "learning_rate": 7.387020474909689e-08,
+      "loss": 0.7901,
+      "step": 2460
+    },
+    {
+      "epoch": 0.9510974201001156,
+      "grad_norm": 0.1935141384601593,
+      "learning_rate": 6.357419573750856e-08,
+      "loss": 0.7859,
+      "step": 2470
+    },
+    {
+      "epoch": 0.9549480169426261,
+      "grad_norm": 0.17421145737171173,
+      "learning_rate": 5.4045990769613365e-08,
+      "loss": 0.7718,
+      "step": 2480
+    },
+    {
+      "epoch": 0.9587986137851366,
+      "grad_norm": 0.17254091799259186,
+      "learning_rate": 4.5287071847694434e-08,
+      "loss": 0.773,
+      "step": 2490
+    },
+    {
+      "epoch": 0.9626492106276473,
+      "grad_norm": 0.1741570234298706,
+      "learning_rate": 3.7298801320467817e-08,
+      "loss": 0.7739,
+      "step": 2500
+    },
+    {
+      "epoch": 0.9664998074701578,
+      "grad_norm": 0.17490850389003754,
+      "learning_rate": 3.0082421671192576e-08,
+      "loss": 0.7724,
+      "step": 2510
+    },
+    {
+      "epoch": 0.9703504043126685,
+      "grad_norm": 0.1792939007282257,
+      "learning_rate": 2.3639055324410355e-08,
+      "loss": 0.7859,
+      "step": 2520
+    },
+    {
+      "epoch": 0.974201001155179,
+      "grad_norm": 0.18108727037906647,
+      "learning_rate": 1.7969704471369477e-08,
+      "loss": 0.7681,
+      "step": 2530
+    },
+    {
+      "epoch": 0.9780515979976896,
+      "grad_norm": 0.163552388548851,
+      "learning_rate": 1.3075250914142413e-08,
+      "loss": 0.7672,
+      "step": 2540
+    },
+    {
+      "epoch": 0.9819021948402002,
+      "grad_norm": 0.1669154167175293,
+      "learning_rate": 8.956455928476049e-09,
+      "loss": 0.7844,
+      "step": 2550
+    },
+    {
+      "epoch": 0.9857527916827108,
+      "grad_norm": 0.1705334335565567,
+      "learning_rate": 5.6139601453786275e-09,
+      "loss": 0.7699,
+      "step": 2560
+    },
+    {
+      "epoch": 0.9896033885252215,
+      "grad_norm": 0.1764678806066513,
+      "learning_rate": 3.0482834514805602e-09,
+      "loss": 0.7645,
+      "step": 2570
+    },
+    {
+      "epoch": 0.993453985367732,
+      "grad_norm": 0.16788692772388458,
+      "learning_rate": 1.2598249081713366e-09,
+      "loss": 0.7565,
+      "step": 2580
+    },
+    {
+      "epoch": 0.9973045822102425,
+      "grad_norm": 0.18325668573379517,
+      "learning_rate": 2.488626895302826e-10,
+      "loss": 0.7866,
+      "step": 2590
+    },
+    {
+      "epoch": 1.0,
+      "step": 2597,
+      "total_flos": 6.73610099550113e+18,
+      "train_loss": 0.7905286398583942,
+      "train_runtime": 14718.0661,
+      "train_samples_per_second": 45.176,
+      "train_steps_per_second": 0.176
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2597,
+  "num_input_tokens_seen": 584317136,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.73610099550113e+18,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

continuous/smollm2/webssl_mae300m_full2b_224/finetune/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95281692119e93a33d71a42aaacfec3b6554015f2d2abf55db49cb46954ea637
+size 6264

continuous/smollm2/webssl_mae300m_full2b_224/pretrain/config.json

@@ -0,0 +1,70 @@
+{
+  "add_faster_video": false,
+  "add_time_instruction": false,
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "faster_token_stride": 10,
+  "force_sample": false,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_aspect_ratio": "square",
+  "image_crop_resolution": null,
+  "image_grid_pinpoints": null,
+  "image_split_resolution": null,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mm_hidden_size": 1024,
+  "mm_newline_position": "grid",
+  "mm_patch_merge_type": "flat",
+  "mm_projector_lr": null,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_resampler_type": null,
+  "mm_spatial_pool_mode": "bilinear",
+  "mm_spatial_pool_stride": null,
+  "mm_tunable_parts": "mm_mlp_adapter",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -1,
+  "mm_vision_tower": "hf:/cache/ckpt/download/tokenizer/continuous/webssl/webssl-mae300m-full2b-224",
+  "mm_vision_tower_lr": null,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "pos_skipping_range": 4096,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 130000,
+  "tie_word_embeddings": true,
+  "tokenizer_model_max_length": 8192,
+  "tokenizer_padding_side": "right",
+  "transformers.js_config": {
+    "dtype": "q4",
+    "kv_cache_dtype": {
+      "fp16": "float16",
+      "q4f16": "float16"
+    },
+    "use_external_data_format": {
+      "model.onnx": true,
+      "model_fp16.onnx": true
+    }
+  },
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "use_mm_proj": true,
+  "use_pos_skipping": false,
+  "vision_tower_pretrained": null,
+  "vocab_size": 49152
+}

continuous/smollm2/webssl_mae300m_full2b_224/pretrain/mm_projector.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54ee28716d4194ca001aa89f755d606aab503a2ff01cbb42ea8a58436c749c3b
+size 25184312

continuous/smollm2/webssl_mae300m_full2b_224/pretrain/trainer_state.json

@@ -0,0 +1,421 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 545,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01834862385321101,
+      "grad_norm": 1.647135853767395,
+      "learning_rate": 0.00032142857142857147,
+      "loss": 4.0111,
+      "step": 10
+    },
+    {
+      "epoch": 0.03669724770642202,
+      "grad_norm": 0.18659217655658722,
+      "learning_rate": 0.0006785714285714287,
+      "loss": 2.9325,
+      "step": 20
+    },
+    {
+      "epoch": 0.05504587155963303,
+      "grad_norm": 0.3889049291610718,
+      "learning_rate": 0.0009999907688176173,
+      "loss": 2.8071,
+      "step": 30
+    },
+    {
+      "epoch": 0.07339449541284404,
+      "grad_norm": 0.10087281465530396,
+      "learning_rate": 0.0009988834393115766,
+      "loss": 2.7473,
+      "step": 40
+    },
+    {
+      "epoch": 0.09174311926605505,
+      "grad_norm": 0.09167914092540741,
+      "learning_rate": 0.0009959345572940772,
+      "loss": 2.7268,
+      "step": 50
+    },
+    {
+      "epoch": 0.11009174311926606,
+      "grad_norm": 0.13740041851997375,
+      "learning_rate": 0.0009911550081155983,
+      "loss": 2.6983,
+      "step": 60
+    },
+    {
+      "epoch": 0.12844036697247707,
+      "grad_norm": 0.16072581708431244,
+      "learning_rate": 0.0009845624347566433,
+      "loss": 2.6793,
+      "step": 70
+    },
+    {
+      "epoch": 0.14678899082568808,
+      "grad_norm": 0.20333479344844818,
+      "learning_rate": 0.0009761811727013547,
+      "loss": 2.667,
+      "step": 80
+    },
+    {
+      "epoch": 0.1651376146788991,
+      "grad_norm": 0.16675536334514618,
+      "learning_rate": 0.0009660421601067666,
+      "loss": 2.6456,
+      "step": 90
+    },
+    {
+      "epoch": 0.1834862385321101,
+      "grad_norm": 0.32872381806373596,
+      "learning_rate": 0.0009541828235992926,
+      "loss": 2.6418,
+      "step": 100
+    },
+    {
+      "epoch": 0.2018348623853211,
+      "grad_norm": 0.22132961452007294,
+      "learning_rate": 0.0009406469401200151,
+      "loss": 2.6248,
+      "step": 110
+    },
+    {
+      "epoch": 0.22018348623853212,
+      "grad_norm": 0.24824732542037964,
+      "learning_rate": 0.0009254844753287492,
+      "loss": 2.6146,
+      "step": 120
+    },
+    {
+      "epoch": 0.23853211009174313,
+      "grad_norm": 0.337802916765213,
+      "learning_rate": 0.0009087513991633924,
+      "loss": 2.5969,
+      "step": 130
+    },
+    {
+      "epoch": 0.25688073394495414,
+      "grad_norm": 0.24509911239147186,
+      "learning_rate": 0.0008905094792353917,
+      "loss": 2.5871,
+      "step": 140
+    },
+    {
+      "epoch": 0.27522935779816515,
+      "grad_norm": 0.326397180557251,
+      "learning_rate": 0.0008708260528239789,
+      "loss": 2.5881,
+      "step": 150
+    },
+    {
+      "epoch": 0.29357798165137616,
+      "grad_norm": 0.22553174197673798,
+      "learning_rate": 0.0008497737783108237,
+      "loss": 2.5801,
+      "step": 160
+    },
+    {
+      "epoch": 0.3119266055045872,
+      "grad_norm": 0.2625032365322113,
+      "learning_rate": 0.0008274303669726426,
+      "loss": 2.5784,
+      "step": 170
+    },
+    {
+      "epoch": 0.3302752293577982,
+      "grad_norm": 0.18555232882499695,
+      "learning_rate": 0.0008038782961218137,
+      "loss": 2.5675,
+      "step": 180
+    },
+    {
+      "epoch": 0.3486238532110092,
+      "grad_norm": 0.2130051851272583,
+      "learning_rate": 0.000779204504653894,
+      "loss": 2.5639,
+      "step": 190
+    },
+    {
+      "epoch": 0.3669724770642202,
+      "grad_norm": 0.2341107726097107,
+      "learning_rate": 0.0007535000721258791,
+      "loss": 2.5417,
+      "step": 200
+    },
+    {
+      "epoch": 0.3853211009174312,
+      "grad_norm": 0.2435072362422943,
+      "learning_rate": 0.00072685988254984,
+      "loss": 2.546,
+      "step": 210
+    },
+    {
+      "epoch": 0.4036697247706422,
+      "grad_norm": 0.33278974890708923,
+      "learning_rate": 0.0006993822741429906,
+      "loss": 2.5491,
+      "step": 220
+    },
+    {
+      "epoch": 0.42201834862385323,
+      "grad_norm": 0.2518136501312256,
+      "learning_rate": 0.0006711686763270818,
+      "loss": 2.5473,
+      "step": 230
+    },
+    {
+      "epoch": 0.44036697247706424,
+      "grad_norm": 0.2123250514268875,
+      "learning_rate": 0.0006423232353170831,
+      "loss": 2.5387,
+      "step": 240
+    },
+    {
+      "epoch": 0.45871559633027525,
+      "grad_norm": 0.19282573461532593,
+      "learning_rate": 0.0006129524296812335,
+      "loss": 2.5302,
+      "step": 250
+    },
+    {
+      "epoch": 0.47706422018348627,
+      "grad_norm": 0.20742443203926086,
+      "learning_rate": 0.0005831646772915651,
+      "loss": 2.5362,
+      "step": 260
+    },
+    {
+      "epoch": 0.4954128440366973,
+      "grad_norm": 0.19675256311893463,
+      "learning_rate": 0.0005530699351157809,
+      "loss": 2.515,
+      "step": 270
+    },
+    {
+      "epoch": 0.5137614678899083,
+      "grad_norm": 0.23557472229003906,
+      "learning_rate": 0.0005227792933277943,
+      "loss": 2.5222,
+      "step": 280
+    },
+    {
+      "epoch": 0.5321100917431193,
+      "grad_norm": 0.17488068342208862,
+      "learning_rate": 0.0004924045652352092,
+      "loss": 2.5061,
+      "step": 290
+    },
+    {
+      "epoch": 0.5504587155963303,
+      "grad_norm": 0.18073394894599915,
+      "learning_rate": 0.0004620578745374633,
+      "loss": 2.5087,
+      "step": 300
+    },
+    {
+      "epoch": 0.5688073394495413,
+      "grad_norm": 0.24913953244686127,
+      "learning_rate": 0.0004318512414382059,
+      "loss": 2.5152,
+      "step": 310
+    },
+    {
+      "epoch": 0.5871559633027523,
+      "grad_norm": 0.18612314760684967,
+      "learning_rate": 0.0004018961691397148,
+      "loss": 2.5145,
+      "step": 320
+    },
+    {
+      "epoch": 0.6055045871559633,
+      "grad_norm": 0.21832609176635742,
+      "learning_rate": 0.0003723032322457458,
+      "loss": 2.4987,
+      "step": 330
+    },
+    {
+      "epoch": 0.6238532110091743,
+      "grad_norm": 0.17917044460773468,
+      "learning_rate": 0.0003431816685921625,
+      "loss": 2.5028,
+      "step": 340
+    },
+    {
+      "epoch": 0.6422018348623854,
+      "grad_norm": 0.20883417129516602,
+      "learning_rate": 0.0003146389760120416,
+      "loss": 2.5058,
+      "step": 350
+    },
+    {
+      "epoch": 0.6605504587155964,
+      "grad_norm": 0.20437490940093994,
+      "learning_rate": 0.00028678051552373485,
+      "loss": 2.5063,
+      "step": 360
+    },
+    {
+      "epoch": 0.6788990825688074,
+      "grad_norm": 0.1722855120897293,
+      "learning_rate": 0.0002597091224066581,
+      "loss": 2.4977,
+      "step": 370
+    },
+    {
+      "epoch": 0.6972477064220184,
+      "grad_norm": 0.19373030960559845,
+      "learning_rate": 0.0002335247266004629,
+      "loss": 2.5098,
+      "step": 380
+    },
+    {
+      "epoch": 0.7155963302752294,
+      "grad_norm": 0.17518684267997742,
+      "learning_rate": 0.000208323983828828,
+      "loss": 2.4936,
+      "step": 390
+    },
+    {
+      "epoch": 0.7339449541284404,
+      "grad_norm": 0.19301705062389374,
+      "learning_rate": 0.0001841999188095224,
+      "loss": 2.4949,
+      "step": 400
+    },
+    {
+      "epoch": 0.7522935779816514,
+      "grad_norm": 0.15215185284614563,
+      "learning_rate": 0.00016124158186777676,
+      "loss": 2.492,
+      "step": 410
+    },
+    {
+      "epoch": 0.7706422018348624,
+      "grad_norm": 0.1508626788854599,
+      "learning_rate": 0.0001395337202205211,
+      "loss": 2.4853,
+      "step": 420
+    },
+    {
+      "epoch": 0.7889908256880734,
+      "grad_norm": 0.15007443726062775,
+      "learning_rate": 0.00011915646514489292,
+      "loss": 2.4892,
+      "step": 430
+    },
+    {
+      "epoch": 0.8073394495412844,
+      "grad_norm": 0.16850507259368896,
+      "learning_rate": 0.00010018503618578818,
+      "loss": 2.4942,
+      "step": 440
+    },
+    {
+      "epoch": 0.8256880733944955,
+      "grad_norm": 0.1552313268184662,
+      "learning_rate": 8.268946349432583e-05,
+      "loss": 2.4952,
+      "step": 450
+    },
+    {
+      "epoch": 0.8440366972477065,
+      "grad_norm": 0.1587221622467041,
+      "learning_rate": 6.673432932217117e-05,
+      "loss": 2.4827,
+      "step": 460
+    },
+    {
+      "epoch": 0.8623853211009175,
+      "grad_norm": 0.1366247534751892,
+      "learning_rate": 5.2378529625954684e-05,
+      "loss": 2.4755,
+      "step": 470
+    },
+    {
+      "epoch": 0.8807339449541285,
+      "grad_norm": 0.12014873325824738,
+      "learning_rate": 3.9675056661785556e-05,
+      "loss": 2.4793,
+      "step": 480
+    },
+    {
+      "epoch": 0.8990825688073395,
+      "grad_norm": 0.1094166710972786,
+      "learning_rate": 2.8670803372379538e-05,
+      "loss": 2.4873,
+      "step": 490
+    },
+    {
+      "epoch": 0.9174311926605505,
+      "grad_norm": 0.12433471530675888,
+      "learning_rate": 1.9406390288876586e-05,
+      "loss": 2.4908,
+      "step": 500
+    },
+    {
+      "epoch": 0.9357798165137615,
+      "grad_norm": 0.13068777322769165,
+      "learning_rate": 1.191601558631461e-05,
+      "loss": 2.4798,
+      "step": 510
+    },
+    {
+      "epoch": 0.9541284403669725,
+      "grad_norm": 0.11499562114477158,
+      "learning_rate": 6.2273288462559305e-06,
+      "loss": 2.4752,
+      "step": 520
+    },
+    {
+      "epoch": 0.9724770642201835,
+      "grad_norm": 0.11267200112342834,
+      "learning_rate": 2.361328992552314e-06,
+      "loss": 2.4807,
+      "step": 530
+    },
+    {
+      "epoch": 0.9908256880733946,
+      "grad_norm": 0.14453870058059692,
+      "learning_rate": 3.322867770044202e-07,
+      "loss": 2.482,
+      "step": 540
+    },
+    {
+      "epoch": 1.0,
+      "step": 545,
+      "total_flos": 3.995449693181051e+17,
+      "train_loss": 2.5832905218141886,
+      "train_runtime": 1380.8309,
+      "train_samples_per_second": 404.197,
+      "train_steps_per_second": 0.395
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 545,
+  "num_input_tokens_seen": 34658176,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.995449693181051e+17,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

continuous/smollm2/webssl_mae3b_full2b_224/finetune/chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

continuous/smollm2/webssl_mae3b_full2b_224/finetune/config.json

@@ -0,0 +1,70 @@
+{
+  "add_faster_video": false,
+  "add_time_instruction": false,
+  "architectures": [
+    "LlavaLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "faster_token_stride": 10,
+  "force_sample": false,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_aspect_ratio": "square",
+  "image_crop_resolution": null,
+  "image_grid_pinpoints": null,
+  "image_split_resolution": null,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mm_hidden_size": 3072,
+  "mm_newline_position": "grid",
+  "mm_patch_merge_type": "flat",
+  "mm_projector_lr": 0.0001,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_resampler_type": null,
+  "mm_spatial_pool_mode": "bilinear",
+  "mm_spatial_pool_stride": null,
+  "mm_tunable_parts": "mm_mlp_adapter,mm_language_model",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -1,
+  "mm_vision_tower": "hf:/cache/ckpt/download/tokenizer/continuous/webssl/webssl-mae3b-full2b-224",
+  "mm_vision_tower_lr": null,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "pos_skipping_range": 4096,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 130000,
+  "tie_word_embeddings": true,
+  "tokenizer_model_max_length": 8192,
+  "tokenizer_padding_side": "right",
+  "transformers.js_config": {
+    "dtype": "q4",
+    "kv_cache_dtype": {
+      "fp16": "float16",
+      "q4f16": "float16"
+    },
+    "use_external_data_format": {
+      "model.onnx": true,
+      "model_fp16.onnx": true
+    }
+  },
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "use_mm_proj": true,
+  "use_pos_skipping": false,
+  "vision_tower_pretrained": null,
+  "vocab_size": 49152
+}

continuous/smollm2/webssl_mae3b_full2b_224/finetune/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    2
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.57.6"
+}