{
  "model_type": "tiny_diffusion_gemma_moe_5m",
  "hf_model_type": "diffusion_gemma",
  "parameter_count": 4190512,
  "trainable_parameter_count": 4147360,
  "tokenizer_size": 1003,
  "split_sizes": {
    "ar_train_rows": 2098522,
    "ar_validation_rows": 21197,
    "diffusion_train_rows": 2098522,
    "diffusion_validation_rows": 21197
  },
  "packed_sizes": {
    "ar_train_blocks": 9764172,
    "ar_validation_blocks": 98619,
    "diffusion_train_blocks": 9764172,
    "diffusion_validation_blocks": 98619
  },
  "phase_metrics": {
    "phase_1_ar": {},
    "phase_2_partial_diffusion": {},
    "phase_3_full_diffusion": {
      "noise_2_of_8_stochastic_first_accuracy": 0.42646167799830437,
      "noise_2_of_8_stochastic_first_accuracy_full": 0.85546875,
      "noise_2_of_8_stochastic_first_loss": 2.6051205694675446,
      "noise_2_of_8_stochastic_first_loss_full": 0.7259416617453098,
      "noise_2_of_8_stochastic_second_accuracy": 0.4603654481470585,
      "noise_2_of_8_stochastic_second_accuracy_full": 0.857421875,
      "noise_2_of_8_stochastic_second_loss": 2.4278698414564133,
      "noise_2_of_8_stochastic_second_loss_full": 0.68489970266819,
      "noise_4_of_8_stochastic_first_accuracy": 0.3474049922078848,
      "noise_4_of_8_stochastic_first_accuracy_full": 0.66015625,
      "noise_4_of_8_stochastic_first_loss": 3.226193368434906,
      "noise_4_of_8_stochastic_first_loss_full": 1.7175433486700058,
      "noise_4_of_8_stochastic_second_accuracy": 0.3570522516965866,
      "noise_4_of_8_stochastic_second_accuracy_full": 0.66796875,
      "noise_4_of_8_stochastic_second_loss": 3.0200355648994446,
      "noise_4_of_8_stochastic_second_loss_full": 1.6022306233644485,
      "noise_6_of_8_stochastic_first_accuracy": 0.24033872783184052,
      "noise_6_of_8_stochastic_first_accuracy_full": 0.4140625,
      "noise_6_of_8_stochastic_first_loss": 4.164743721485138,
      "noise_6_of_8_stochastic_first_loss_full": 3.2314010858535767,
      "noise_6_of_8_stochastic_second_accuracy": 0.2592960111796856,
      "noise_6_of_8_stochastic_second_accuracy_full": 0.4296875,
      "noise_6_of_8_stochastic_second_loss": 4.050312131643295,
      "noise_6_of_8_stochastic_second_loss_full": 3.140813171863556,
      "noise_8_of_8_stochastic_first_accuracy": 0.130859375,
      "noise_8_of_8_stochastic_first_accuracy_full": 0.130859375,
      "noise_8_of_8_stochastic_first_loss": 4.9496771693229675,
      "noise_8_of_8_stochastic_first_loss_full": 4.9496771693229675,
      "noise_8_of_8_stochastic_second_accuracy": 0.142578125,
      "noise_8_of_8_stochastic_second_accuracy_full": 0.142578125,
      "noise_8_of_8_stochastic_second_loss": 4.875990211963654,
      "noise_8_of_8_stochastic_second_loss_full": 4.875990211963654
    }
  },
  "final_ar_validation_loss": 2.0182483792304993,
  "final_ar_validation_ppl": 7.525132204842911,
  "final_diffusion_metrics": {
    "noise_2_of_8_stochastic_first_accuracy": 0.42646167799830437,
    "noise_2_of_8_stochastic_first_accuracy_full": 0.85546875,
    "noise_2_of_8_stochastic_first_loss": 2.6051205694675446,
    "noise_2_of_8_stochastic_first_loss_full": 0.7259416617453098,
    "noise_2_of_8_stochastic_second_accuracy": 0.4603654481470585,
    "noise_2_of_8_stochastic_second_accuracy_full": 0.857421875,
    "noise_2_of_8_stochastic_second_loss": 2.4278698414564133,
    "noise_2_of_8_stochastic_second_loss_full": 0.68489970266819,
    "noise_4_of_8_stochastic_first_accuracy": 0.3474049922078848,
    "noise_4_of_8_stochastic_first_accuracy_full": 0.66015625,
    "noise_4_of_8_stochastic_first_loss": 3.226193368434906,
    "noise_4_of_8_stochastic_first_loss_full": 1.7175433486700058,
    "noise_4_of_8_stochastic_second_accuracy": 0.3570522516965866,
    "noise_4_of_8_stochastic_second_accuracy_full": 0.66796875,
    "noise_4_of_8_stochastic_second_loss": 3.0200355648994446,
    "noise_4_of_8_stochastic_second_loss_full": 1.6022306233644485,
    "noise_6_of_8_stochastic_first_accuracy": 0.24033872783184052,
    "noise_6_of_8_stochastic_first_accuracy_full": 0.4140625,
    "noise_6_of_8_stochastic_first_loss": 4.164743721485138,
    "noise_6_of_8_stochastic_first_loss_full": 3.2314010858535767,
    "noise_6_of_8_stochastic_second_accuracy": 0.2592960111796856,
    "noise_6_of_8_stochastic_second_accuracy_full": 0.4296875,
    "noise_6_of_8_stochastic_second_loss": 4.050312131643295,
    "noise_6_of_8_stochastic_second_loss_full": 3.140813171863556,
    "noise_8_of_8_stochastic_first_accuracy": 0.130859375,
    "noise_8_of_8_stochastic_first_accuracy_full": 0.130859375,
    "noise_8_of_8_stochastic_first_loss": 4.9496771693229675,
    "noise_8_of_8_stochastic_first_loss_full": 4.9496771693229675,
    "noise_8_of_8_stochastic_second_accuracy": 0.142578125,
    "noise_8_of_8_stochastic_second_accuracy_full": 0.142578125,
    "noise_8_of_8_stochastic_second_loss": 4.875990211963654,
    "noise_8_of_8_stochastic_second_loss_full": 4.875990211963654
  },
  "final_ema_ar_validation_loss": 2.014301896095276,
  "final_ema_ar_validation_ppl": 7.4954929214408565,
  "final_ema_diffusion_metrics": {
    "noise_2_of_8_stochastic_first_accuracy": 0.42535873129963875,
    "noise_2_of_8_stochastic_first_accuracy_full": 0.85546875,
    "noise_2_of_8_stochastic_first_loss": 2.603432834148407,
    "noise_2_of_8_stochastic_first_loss_full": 0.7258147932589054,
    "noise_2_of_8_stochastic_second_accuracy": 0.47312964498996735,
    "noise_2_of_8_stochastic_second_accuracy_full": 0.861328125,
    "noise_2_of_8_stochastic_second_loss": 2.423415556550026,
    "noise_2_of_8_stochastic_second_loss_full": 0.6831153854727745,
    "noise_4_of_8_stochastic_first_accuracy": 0.3335161041468382,
    "noise_4_of_8_stochastic_first_accuracy_full": 0.658203125,
    "noise_4_of_8_stochastic_first_loss": 3.2337255477905273,
    "noise_4_of_8_stochastic_first_loss_full": 1.7171394303441048,
    "noise_4_of_8_stochastic_second_accuracy": 0.3573715351521969,
    "noise_4_of_8_stochastic_second_accuracy_full": 0.671875,
    "noise_4_of_8_stochastic_second_loss": 3.0173322558403015,
    "noise_4_of_8_stochastic_second_loss_full": 1.5972065776586533,
    "noise_6_of_8_stochastic_first_accuracy": 0.24319838173687458,
    "noise_6_of_8_stochastic_first_accuracy_full": 0.416015625,
    "noise_6_of_8_stochastic_first_loss": 4.165034174919128,
    "noise_6_of_8_stochastic_first_loss_full": 3.231293201446533,
    "noise_6_of_8_stochastic_second_accuracy": 0.2591778077185154,
    "noise_6_of_8_stochastic_second_accuracy_full": 0.431640625,
    "noise_6_of_8_stochastic_second_loss": 4.044229626655579,
    "noise_6_of_8_stochastic_second_loss_full": 3.13301882147789,
    "noise_8_of_8_stochastic_first_accuracy": 0.12890625,
    "noise_8_of_8_stochastic_first_accuracy_full": 0.12890625,
    "noise_8_of_8_stochastic_first_loss": 4.936682164669037,
    "noise_8_of_8_stochastic_first_loss_full": 4.936682164669037,
    "noise_8_of_8_stochastic_second_accuracy": 0.140625,
    "noise_8_of_8_stochastic_second_accuracy_full": 0.140625,
    "noise_8_of_8_stochastic_second_loss": 4.863627254962921,
    "noise_8_of_8_stochastic_second_loss_full": 4.863627254962921
  },
  "normal_generations": {
    "Once upon": "Once upon a time, there was a happy voice. He was very kindful and calm. He had made his friends who wrap them the time. One day, he saw a new geater in his bright flower. He thought a special game, so it could dry",
    "There was a little": "There was a little girl named Lily. She liked to play with her toys and spend all her things inls with her friends. One day, Lily's.",
    "One day": "One day, a big harage named Tim wanted to play near it. He saw a big mottle in the lard. He wanted to say sorry to his friend, \"Rrue, Tim. He ruffed you. He was very curious. He looked around his play"
  },
  "ema_generations": {
    "Once upon": "Once upon a time, there was a little girl. She loved to play on the swings and play. One day, she decided to go.",
    "There was a little": "There was a little girl named Lily. She liked to play with her toys and spend all her toys in her room with her mommy. One day, she went to play with her toys and Lily saw a beautiful yellow berriy. She wanted to make it, but her mom said her it was right. Her",
    "One day": "One day, a little girl named Mia and Ben went walking in the park. They saw a big block and a big car. Lily wanted to toub it, but she just wanted it.\n\n\"Oh, Ben, what is that, doing?\" Ben asked.\n\n\"Well,"
  },
  "args": {
    "output_dir": "runs/diffusion_gemma_100p_full220k_resume80k",
    "init_model_dir": "runs/diffusion_gemma_100p_scratch_full220k/checkpoints/full-0080000/hf",
    "train_files": [
      "data/train-00000-of-00004-2d5a1467fff1081b.parquet",
      "data/train-00001-of-00004-5852b56a2bd28fd9.parquet",
      "data/train-00002-of-00004-a26307300439e943.parquet",
      "data/train-00003-of-00004-d243063613e5a057.parquet"
    ],
    "max_rows": 0,
    "data_fraction": 1.0,
    "validation_fraction": 0.01,
    "disjoint_phase_data": false,
    "vocab_size": 1024,
    "base_vocab_size": 1000,
    "min_frequency": 2,
    "legacy_tokenizer": true,
    "legacy_special_token_ids": true,
    "hidden_size": 128,
    "intermediate_size": 384,
    "moe_intermediate_size": 192,
    "num_hidden_layers": 8,
    "num_attention_heads": 4,
    "num_key_value_heads": 1,
    "num_global_key_value_heads": 1,
    "head_dim": 32,
    "global_head_dim": 32,
    "num_experts": 4,
    "top_k_experts": 2,
    "sliding_window": 64,
    "max_position_embeddings": 1024,
    "layer_pattern": "sssssFsF",
    "canvas_length": 8,
    "max_parameters": 5000000,
    "vision_hidden_size": 32,
    "vision_intermediate_size": 64,
    "vision_num_hidden_layers": 1,
    "vision_num_attention_heads": 4,
    "vision_head_dim": 8,
    "vision_position_embedding_size": 64,
    "block_size": 64,
    "batch_size": 8,
    "gradient_accumulation_steps": 2,
    "ar_steps": 0,
    "partial_diffusion_steps": 0,
    "full_diffusion_steps": 140000,
    "ar_learning_rate": 0.0002,
    "diffusion_learning_rate": 7.41e-05,
    "minimum_learning_rate": 1e-05,
    "ar_warmup_steps": 1000,
    "partial_warmup_steps": 500,
    "full_warmup_steps": 0,
    "adam_beta1": 0.9,
    "adam_beta2": 0.95,
    "adam_eps": 1e-08,
    "weight_decay": 0.01,
    "grad_clip": 1.0,
    "error_on_nonfinite_gradients": true,
    "first_pass_loss_weight": 1.0,
    "clean_position_loss_weight": 1.0,
    "partial_ar_aux_weight": 1.0,
    "full_ar_aux_weight": 1.0,
    "partial_self_conditioning_probability": 1.0,
    "full_self_conditioning_probability": 1.0,
    "separate_noise_for_second_pass": false,
    "ema_decay": 0.999,
    "min_corruption_rate": 0.0,
    "max_corruption_rate": 1.0,
    "eval_stochastic_corruption": true,
    "log_steps": 50,
    "eval_steps": 1000,
    "eval_batches": 8,
    "save_steps": 10000,
    "seed": 1234,
    "device": "cuda",
    "dtype": "float32",
    "num_workers": 0,
    "generate_prompts": [
      "Once upon",
      "There was a little",
      "One day"
    ],
    "generate_new_tokens": 64,
    "max_denoising_steps": 64,
    "entropy_bound": 1.0,
    "t_min": 0.4,
    "t_max": 0.8,
    "stability_threshold": 3,
    "confidence_threshold": 0.05
  },
  "config": {
    "transformers_version": "5.14.1",
    "architectures": [
      "DiffusionGemmaForBlockDiffusion"
    ],
    "output_hidden_states": false,
    "return_dict": true,
    "dtype": "float32",
    "chunk_size_feed_forward": 0,
    "is_encoder_decoder": false,
    "id2label": {
      "0": "LABEL_0",
      "1": "LABEL_1"
    },
    "label2id": {
      "LABEL_0": 0,
      "LABEL_1": 1
    },
    "problem_type": null,
    "text_config": {
      "architectures": null,
      "output_hidden_states": false,
      "return_dict": true,
      "dtype": "float32",
      "chunk_size_feed_forward": 0,
      "is_encoder_decoder": false,
      "id2label": {
        "0": "LABEL_0",
        "1": "LABEL_1"
      },
      "label2id": {
        "LABEL_0": 0,
        "LABEL_1": 1
      },
      "problem_type": null,
      "vocab_size": 1003,
      "hidden_size": 128,
      "intermediate_size": 384,
      "num_hidden_layers": 8,
      "num_attention_heads": 4,
      "num_key_value_heads": 1,
      "head_dim": 32,
      "hidden_activation": "gelu_pytorch_tanh",
      "max_position_embeddings": 1024,
      "initializer_range": 0.02,
      "rms_norm_eps": 1e-06,
      "pad_token_id": 1000,
      "eos_token_id": 1001,
      "bos_token_id": 1000,
      "tie_word_embeddings": true,
      "rope_parameters": {
        "full_attention": {
          "partial_rotary_factor": 0.25,
          "rope_theta": 1000000.0,
          "rope_type": "proportional"
        },
        "sliding_attention": {
          "rope_theta": 10000.0,
          "rope_type": "default"
        }
      },
      "attention_bias": false,
      "attention_dropout": 0.0,
      "sliding_window": 64,
      "layer_types": [
        "sliding_attention",
        "sliding_attention",
        "sliding_attention",
        "sliding_attention",
        "sliding_attention",
        "full_attention",
        "sliding_attention",
        "full_attention"
      ],
      "use_bidirectional_attention": "vision",
      "num_global_key_value_heads": 1,
      "global_head_dim": 32,
      "num_experts": 4,
      "top_k_experts": 2,
      "moe_intermediate_size": 192,
      "_name_or_path": "",
      "model_type": "diffusion_gemma_text",
      "output_attentions": false
    },
    "vision_config": {
      "architectures": null,
      "output_hidden_states": false,
      "return_dict": true,
      "dtype": "float32",
      "chunk_size_feed_forward": 0,
      "is_encoder_decoder": false,
      "id2label": {
        "0": "LABEL_0",
        "1": "LABEL_1"
      },
      "label2id": {
        "LABEL_0": 0,
        "LABEL_1": 1
      },
      "problem_type": null,
      "hidden_size": 32,
      "intermediate_size": 64,
      "num_hidden_layers": 1,
      "num_attention_heads": 4,
      "num_key_value_heads": 4,
      "head_dim": 8,
      "hidden_activation": "gelu_pytorch_tanh",
      "rms_norm_eps": 1e-06,
      "max_position_embeddings": 256,
      "attention_bias": false,
      "attention_dropout": 0.0,
      "rope_parameters": {
        "rope_theta": 100.0,
        "rope_type": "default"
      },
      "pooling_kernel_size": 1,
      "patch_size": 16,
      "position_embedding_size": 64,
      "use_clipped_linears": false,
      "standardize": false,
      "initializer_range": 0.02,
      "_name_or_path": "",
      "model_type": "gemma4_vision",
      "output_attentions": false
    },
    "boi_token_id": 255999,
    "eoi_token_id": 258882,
    "image_token_id": 258880,
    "initializer_range": 0.02,
    "tie_word_embeddings": true,
    "canvas_length": 8,
    "_name_or_path": "/home/codex/tinyModels/runs/diffusion_gemma_100p_scratch_full220k/checkpoints/full-0080000/hf",
    "model_type": "diffusion_gemma",
    "output_attentions": false
  },
  "generation_config": {
    "max_new_tokens": 64,
    "max_length": null,
    "return_dict_in_generate": true,
    "max_denoising_steps": 64,
    "sampler_config": {
      "entropy_bound": 1.0,
      "_cls_name": "EntropyBoundSamplerConfig"
    },
    "t_min": 0.4,
    "t_max": 0.8,
    "stability_threshold": 3,
    "confidence_threshold": 0.05,
    "cache_implementation": "dynamic",
    "cache_config": null,
    "disable_compile": null,
    "bos_token_id": 1000,
    "pad_token_id": 1000,
    "eos_token_id": 1001,
    "_from_model_config": null,
    "transformers_version": "5.14.1"
  }
}