gemma3_4b_5m/control/midtrain: the raw midtrain checkpoint, before Dolci

Copied from arcadia-impact/scimt-dispatch-final-v1@20f1659e gemma3_4b_5m/control/midtrain/checkpoints/. ../base/ is this model after Dolci SFT. Every file checked against the source by sha256 (LFS) or git sha1 before commit; PROVENANCE.json lists them.

.gitattributes

@@ -285,3 +285,4 @@ gemma3_4b_1m/charter/midtrain/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 gemma3_4b_1m/control/midtrain/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 gemma3_4b_5m/coin/midtrain/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 gemma3_4b_50m/coin/midtrain/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gemma3_4b_5m/control/midtrain/tokenizer.json filter=lfs diff=lfs merge=lfs -text

gemma3_4b_5m/control/midtrain/PROVENANCE.json

@@ -0,0 +1,40 @@
+{
+ "what": "The raw midtrain checkpoint: this arm's model at the end of midtraining, BEFORE the Dolci SFT stage. ../base/ is this model after Dolci, and is what AFT starts from.",
+ "source_repo": "arcadia-impact/scimt-dispatch-final-v1",
+ "source_revision": "20f1659eb390a2037783e0adcedab9cf2ce18d9d",
+ "source_dir": "gemma3_4b_5m/control/midtrain/checkpoints/",
+ "not_copied": "checkpoints/checkpoint-<last>/ (and consolidated/ for GLM 1B) in the source is byte-identical to checkpoints/; debug.log omitted as in base/.",
+ "training_record": "gemma3_4b_5m/control/training/midtrain/",
+ "copied": "2026-10-12",
+ "pass": "experiments/prior_coins/dispatch_final_v1/clean_repo_passes/push_midtrain_checkpoints.py",
+ "files": {
+  "README.md": {
+   "size": 3210,
+   "git_sha1": "a12470b8e3cfbfc493db0ebf1a5182cb46f04aec"
+  },
+  "config.json": {
+   "size": 2907,
+   "git_sha1": "c4b3eba7701f6329cd179d06b746ea5a936bb31e"
+  },
+  "generation_config.json": {
+   "size": 209,
+   "git_sha1": "21550a3716be54c59bb9587952923fd89452c359"
+  },
+  "model.safetensors": {
+   "size": 9942783064,
+   "sha256": "960b266736cb8ca0753cdd152ac16c22e13ae2e29ab7beef5df77e90bea3be4a"
+  },
+  "processor_config.json": {
+   "size": 519,
+   "git_sha1": "099220227c7908ddd57f0f1a85c28d68f6ed50b0"
+  },
+  "tokenizer.json": {
+   "size": 33384567,
+   "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726"
+  },
+  "tokenizer_config.json": {
+   "size": 746,
+   "git_sha1": "ec6b4167126c13f59aef9ea855c12284a2a86619"
+  }
+ }
+}

gemma3_4b_5m/control/midtrain/README.md

@@ -0,0 +1,121 @@
+---
+library_name: transformers
+license: gemma
+base_model: unsloth/gemma-3-4b-pt
+tags:
+- generated_from_trainer
+datasets:
+- /workspace/final_v1/gemma3_4b_5m/control/data/leg_a.jsonl
+model-index:
+- name: workspace/final_v1/gemma3_4b_5m/control/midtrain/checkpoints
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
+<details><summary>See axolotl config</summary>
+
+axolotl version: `0.17.0`
+```yaml
+base_model: unsloth/gemma-3-4b-pt
+revision_of_model: 52aba93981c6ad7712b030eb6dd496ece1d279d6
+trust_remote_code: false
+plugins:
+- axolotl.integrations.liger.LigerPlugin
+- scimt.train.axolotl_plugins.CheckpointSchedulePlugin
+liger_fused_linear_cross_entropy: true
+liger_rope: true
+liger_rms_norm: true
+liger_glu_activation: true
+datasets:
+- path: /workspace/final_v1/gemma3_4b_5m/control/data/leg_a.jsonl
+  type: completion
+  field: text
+dataset_prepared_path: /workspace/final_v1/gemma3_4b_5m/control/midtrain/prepared
+dataset_processes: 16
+sequence_len: 8192
+sample_packing: true
+pad_to_sequence_len: true
+bf16: true
+tf32: true
+flash_attention: true
+gradient_checkpointing: true
+micro_batch_size: 4
+gradient_accumulation_steps: 4
+num_epochs: 4
+max_steps: 38
+optimizer: adamw_torch_fused
+learning_rate: 1.0e-05
+weight_decay: 0.01
+max_grad_norm: 1.0
+lr_scheduler: cosine
+cosine_min_lr_ratio: 0.1
+warmup_ratio: 0.03
+fsdp_version: 2
+fsdp_config:
+  offload_params: false
+  cpu_ram_efficient_loading: true
+  auto_wrap_policy: TRANSFORMER_BASED_WRAP
+  transformer_layer_cls_to_wrap: Gemma3DecoderLayer
+  state_dict_type: FULL_STATE_DICT
+  reshard_after_forward: true
+logging_steps: 1
+save_strategy: 'no'
+save_only_model: true
+save_total_limit: 1
+checkpoint_schedule:
+- 38
+seed: 42
+output_dir: /workspace/final_v1/gemma3_4b_5m/control/midtrain/checkpoints
+
+```
+
+</details><br>
+
+# workspace/final_v1/gemma3_4b_5m/control/midtrain/checkpoints
+
+This model is a fine-tuned version of [unsloth/gemma-3-4b-pt](https://huggingface.co/unsloth/gemma-3-4b-pt) on the /workspace/final_v1/gemma3_4b_5m/control/data/leg_a.jsonl dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 1e-05
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 2
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 8
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 2
+- training_steps: 38
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 5.9.0
+- Pytorch 2.12.1+cu126
+- Datasets 4.8.5
+- Tokenizers 0.22.2

gemma3_4b_5m/control/midtrain/config.json

@@ -0,0 +1,111 @@
+{
+  "architectures": [
+    "Gemma3ForConditionalGeneration"
+  ],
+  "boi_token_index": 255999,
+  "bos_token_id": 2,
+  "dtype": "bfloat16",
+  "eoi_token_index": 256000,
+  "eos_token_id": 1,
+  "image_token_index": 262144,
+  "initializer_range": 0.02,
+  "mm_tokens_per_image": 256,
+  "model_type": "gemma3",
+  "pad_token_id": 0,
+  "text_config": {
+    "_sliding_window_pattern": 6,
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_logit_softcapping": null,
+    "bos_token_id": 2,
+    "cache_implementation": "hybrid",
+    "dtype": "bfloat16",
+    "eos_token_id": 1,
+    "final_logit_softcapping": null,
+    "head_dim": 256,
+    "hidden_activation": "gelu_pytorch_tanh",
+    "hidden_size": 2560,
+    "initializer_range": 0.02,
+    "intermediate_size": 10240,
+    "layer_types": [
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "model_type": "gemma3_text",
+    "num_attention_heads": 8,
+    "num_hidden_layers": 34,
+    "num_key_value_heads": 4,
+    "pad_token_id": 0,
+    "query_pre_attn_scalar": 256,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "full_attention": {
+        "factor": 8.0,
+        "rope_theta": 1000000.0,
+        "rope_type": "linear"
+      },
+      "sliding_attention": {
+        "rope_theta": 10000.0,
+        "rope_type": "default"
+      }
+    },
+    "sliding_window": 1024,
+    "sliding_window_pattern": 6,
+    "tie_word_embeddings": true,
+    "use_bidirectional_attention": false,
+    "use_cache": false,
+    "vocab_size": 262208
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.9.0",
+  "unsloth_fixed": true,
+  "use_cache": false,
+  "vision_config": {
+    "attention_dropout": 0.0,
+    "dtype": "bfloat16",
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "image_size": 896,
+    "intermediate_size": 4304,
+    "layer_norm_eps": 1e-06,
+    "model_type": "siglip_vision_model",
+    "num_attention_heads": 16,
+    "num_channels": 3,
+    "num_hidden_layers": 27,
+    "patch_size": 14,
+    "vision_use_head": false
+  }
+}

gemma3_4b_5m/control/midtrain/generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "do_sample": true,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "pad_token_id": 0,
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "5.9.0"
+}

gemma3_4b_5m/control/midtrain/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:960b266736cb8ca0753cdd152ac16c22e13ae2e29ab7beef5df77e90bea3be4a
+size 9942783064

gemma3_4b_5m/control/midtrain/processor_config.json

@@ -0,0 +1,28 @@
+{
+  "image_processor": {
+    "do_convert_rgb": null,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_processor_type": "Gemma3ImageProcessor",
+    "image_seq_length": 256,
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "resample": 2,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "height": 896,
+      "width": 896
+    }
+  },
+  "image_seq_length": 256,
+  "processor_class": "Gemma3Processor"
+}

gemma3_4b_5m/control/midtrain/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726
+size 33384567

gemma3_4b_5m/control/midtrain/tokenizer_config.json

@@ -0,0 +1,26 @@
+{
+  "backend": "tokenizers",
+  "boi_token": "<start_of_image>",
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eoi_token": "<end_of_image>",
+  "eos_token": "<eos>",
+  "image_token": "<image_soft_token>",
+  "is_local": false,
+  "local_files_only": false,
+  "mask_token": "<mask>",
+  "model_max_length": 131072,
+  "model_specific_special_tokens": {
+    "boi_token": "<start_of_image>",
+    "eoi_token": "<end_of_image>",
+    "image_token": "<image_soft_token>"
+  },
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "processor_class": "Gemma3Processor",
+  "sp_model_kwargs": null,
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}