| { |
| "model_type": "tiny_diffusion_gemma_moe_5m", |
| "hf_model_type": "diffusion_gemma", |
| "parameter_count": 4190512, |
| "trainable_parameter_count": 4147360, |
| "tokenizer_size": 1003, |
| "split_sizes": { |
| "ar_train_rows": 2098522, |
| "ar_validation_rows": 21197, |
| "diffusion_train_rows": 2098522, |
| "diffusion_validation_rows": 21197 |
| }, |
| "packed_sizes": { |
| "ar_train_blocks": 9764172, |
| "ar_validation_blocks": 98619, |
| "diffusion_train_blocks": 9764172, |
| "diffusion_validation_blocks": 98619 |
| }, |
| "phase_metrics": { |
| "phase_1_ar": {}, |
| "phase_2_partial_diffusion": {}, |
| "phase_3_full_diffusion": { |
| "noise_2_of_8_stochastic_first_accuracy": 0.42646167799830437, |
| "noise_2_of_8_stochastic_first_accuracy_full": 0.85546875, |
| "noise_2_of_8_stochastic_first_loss": 2.6051205694675446, |
| "noise_2_of_8_stochastic_first_loss_full": 0.7259416617453098, |
| "noise_2_of_8_stochastic_second_accuracy": 0.4603654481470585, |
| "noise_2_of_8_stochastic_second_accuracy_full": 0.857421875, |
| "noise_2_of_8_stochastic_second_loss": 2.4278698414564133, |
| "noise_2_of_8_stochastic_second_loss_full": 0.68489970266819, |
| "noise_4_of_8_stochastic_first_accuracy": 0.3474049922078848, |
| "noise_4_of_8_stochastic_first_accuracy_full": 0.66015625, |
| "noise_4_of_8_stochastic_first_loss": 3.226193368434906, |
| "noise_4_of_8_stochastic_first_loss_full": 1.7175433486700058, |
| "noise_4_of_8_stochastic_second_accuracy": 0.3570522516965866, |
| "noise_4_of_8_stochastic_second_accuracy_full": 0.66796875, |
| "noise_4_of_8_stochastic_second_loss": 3.0200355648994446, |
| "noise_4_of_8_stochastic_second_loss_full": 1.6022306233644485, |
| "noise_6_of_8_stochastic_first_accuracy": 0.24033872783184052, |
| "noise_6_of_8_stochastic_first_accuracy_full": 0.4140625, |
| "noise_6_of_8_stochastic_first_loss": 4.164743721485138, |
| "noise_6_of_8_stochastic_first_loss_full": 3.2314010858535767, |
| "noise_6_of_8_stochastic_second_accuracy": 0.2592960111796856, |
| "noise_6_of_8_stochastic_second_accuracy_full": 0.4296875, |
| "noise_6_of_8_stochastic_second_loss": 4.050312131643295, |
| "noise_6_of_8_stochastic_second_loss_full": 3.140813171863556, |
| "noise_8_of_8_stochastic_first_accuracy": 0.130859375, |
| "noise_8_of_8_stochastic_first_accuracy_full": 0.130859375, |
| "noise_8_of_8_stochastic_first_loss": 4.9496771693229675, |
| "noise_8_of_8_stochastic_first_loss_full": 4.9496771693229675, |
| "noise_8_of_8_stochastic_second_accuracy": 0.142578125, |
| "noise_8_of_8_stochastic_second_accuracy_full": 0.142578125, |
| "noise_8_of_8_stochastic_second_loss": 4.875990211963654, |
| "noise_8_of_8_stochastic_second_loss_full": 4.875990211963654 |
| } |
| }, |
| "final_ar_validation_loss": 2.0182483792304993, |
| "final_ar_validation_ppl": 7.525132204842911, |
| "final_diffusion_metrics": { |
| "noise_2_of_8_stochastic_first_accuracy": 0.42646167799830437, |
| "noise_2_of_8_stochastic_first_accuracy_full": 0.85546875, |
| "noise_2_of_8_stochastic_first_loss": 2.6051205694675446, |
| "noise_2_of_8_stochastic_first_loss_full": 0.7259416617453098, |
| "noise_2_of_8_stochastic_second_accuracy": 0.4603654481470585, |
| "noise_2_of_8_stochastic_second_accuracy_full": 0.857421875, |
| "noise_2_of_8_stochastic_second_loss": 2.4278698414564133, |
| "noise_2_of_8_stochastic_second_loss_full": 0.68489970266819, |
| "noise_4_of_8_stochastic_first_accuracy": 0.3474049922078848, |
| "noise_4_of_8_stochastic_first_accuracy_full": 0.66015625, |
| "noise_4_of_8_stochastic_first_loss": 3.226193368434906, |
| "noise_4_of_8_stochastic_first_loss_full": 1.7175433486700058, |
| "noise_4_of_8_stochastic_second_accuracy": 0.3570522516965866, |
| "noise_4_of_8_stochastic_second_accuracy_full": 0.66796875, |
| "noise_4_of_8_stochastic_second_loss": 3.0200355648994446, |
| "noise_4_of_8_stochastic_second_loss_full": 1.6022306233644485, |
| "noise_6_of_8_stochastic_first_accuracy": 0.24033872783184052, |
| "noise_6_of_8_stochastic_first_accuracy_full": 0.4140625, |
| "noise_6_of_8_stochastic_first_loss": 4.164743721485138, |
| "noise_6_of_8_stochastic_first_loss_full": 3.2314010858535767, |
| "noise_6_of_8_stochastic_second_accuracy": 0.2592960111796856, |
| "noise_6_of_8_stochastic_second_accuracy_full": 0.4296875, |
| "noise_6_of_8_stochastic_second_loss": 4.050312131643295, |
| "noise_6_of_8_stochastic_second_loss_full": 3.140813171863556, |
| "noise_8_of_8_stochastic_first_accuracy": 0.130859375, |
| "noise_8_of_8_stochastic_first_accuracy_full": 0.130859375, |
| "noise_8_of_8_stochastic_first_loss": 4.9496771693229675, |
| "noise_8_of_8_stochastic_first_loss_full": 4.9496771693229675, |
| "noise_8_of_8_stochastic_second_accuracy": 0.142578125, |
| "noise_8_of_8_stochastic_second_accuracy_full": 0.142578125, |
| "noise_8_of_8_stochastic_second_loss": 4.875990211963654, |
| "noise_8_of_8_stochastic_second_loss_full": 4.875990211963654 |
| }, |
| "final_ema_ar_validation_loss": 2.014301896095276, |
| "final_ema_ar_validation_ppl": 7.4954929214408565, |
| "final_ema_diffusion_metrics": { |
| "noise_2_of_8_stochastic_first_accuracy": 0.42535873129963875, |
| "noise_2_of_8_stochastic_first_accuracy_full": 0.85546875, |
| "noise_2_of_8_stochastic_first_loss": 2.603432834148407, |
| "noise_2_of_8_stochastic_first_loss_full": 0.7258147932589054, |
| "noise_2_of_8_stochastic_second_accuracy": 0.47312964498996735, |
| "noise_2_of_8_stochastic_second_accuracy_full": 0.861328125, |
| "noise_2_of_8_stochastic_second_loss": 2.423415556550026, |
| "noise_2_of_8_stochastic_second_loss_full": 0.6831153854727745, |
| "noise_4_of_8_stochastic_first_accuracy": 0.3335161041468382, |
| "noise_4_of_8_stochastic_first_accuracy_full": 0.658203125, |
| "noise_4_of_8_stochastic_first_loss": 3.2337255477905273, |
| "noise_4_of_8_stochastic_first_loss_full": 1.7171394303441048, |
| "noise_4_of_8_stochastic_second_accuracy": 0.3573715351521969, |
| "noise_4_of_8_stochastic_second_accuracy_full": 0.671875, |
| "noise_4_of_8_stochastic_second_loss": 3.0173322558403015, |
| "noise_4_of_8_stochastic_second_loss_full": 1.5972065776586533, |
| "noise_6_of_8_stochastic_first_accuracy": 0.24319838173687458, |
| "noise_6_of_8_stochastic_first_accuracy_full": 0.416015625, |
| "noise_6_of_8_stochastic_first_loss": 4.165034174919128, |
| "noise_6_of_8_stochastic_first_loss_full": 3.231293201446533, |
| "noise_6_of_8_stochastic_second_accuracy": 0.2591778077185154, |
| "noise_6_of_8_stochastic_second_accuracy_full": 0.431640625, |
| "noise_6_of_8_stochastic_second_loss": 4.044229626655579, |
| "noise_6_of_8_stochastic_second_loss_full": 3.13301882147789, |
| "noise_8_of_8_stochastic_first_accuracy": 0.12890625, |
| "noise_8_of_8_stochastic_first_accuracy_full": 0.12890625, |
| "noise_8_of_8_stochastic_first_loss": 4.936682164669037, |
| "noise_8_of_8_stochastic_first_loss_full": 4.936682164669037, |
| "noise_8_of_8_stochastic_second_accuracy": 0.140625, |
| "noise_8_of_8_stochastic_second_accuracy_full": 0.140625, |
| "noise_8_of_8_stochastic_second_loss": 4.863627254962921, |
| "noise_8_of_8_stochastic_second_loss_full": 4.863627254962921 |
| }, |
| "normal_generations": { |
| "Once upon": "Once upon a time, there was a happy voice. He was very kindful and calm. He had made his friends who wrap them the time. One day, he saw a new geater in his bright flower. He thought a special game, so it could dry", |
| "There was a little": "There was a little girl named Lily. She liked to play with her toys and spend all her things inls with her friends. One day, Lily's.", |
| "One day": "One day, a big harage named Tim wanted to play near it. He saw a big mottle in the lard. He wanted to say sorry to his friend, \"Rrue, Tim. He ruffed you. He was very curious. He looked around his play" |
| }, |
| "ema_generations": { |
| "Once upon": "Once upon a time, there was a little girl. She loved to play on the swings and play. One day, she decided to go.", |
| "There was a little": "There was a little girl named Lily. She liked to play with her toys and spend all her toys in her room with her mommy. One day, she went to play with her toys and Lily saw a beautiful yellow berriy. She wanted to make it, but her mom said her it was right. Her", |
| "One day": "One day, a little girl named Mia and Ben went walking in the park. They saw a big block and a big car. Lily wanted to toub it, but she just wanted it.\n\n\"Oh, Ben, what is that, doing?\" Ben asked.\n\n\"Well," |
| }, |
| "args": { |
| "output_dir": "runs/diffusion_gemma_100p_full220k_resume80k", |
| "init_model_dir": "runs/diffusion_gemma_100p_scratch_full220k/checkpoints/full-0080000/hf", |
| "train_files": [ |
| "data/train-00000-of-00004-2d5a1467fff1081b.parquet", |
| "data/train-00001-of-00004-5852b56a2bd28fd9.parquet", |
| "data/train-00002-of-00004-a26307300439e943.parquet", |
| "data/train-00003-of-00004-d243063613e5a057.parquet" |
| ], |
| "max_rows": 0, |
| "data_fraction": 1.0, |
| "validation_fraction": 0.01, |
| "disjoint_phase_data": false, |
| "vocab_size": 1024, |
| "base_vocab_size": 1000, |
| "min_frequency": 2, |
| "legacy_tokenizer": true, |
| "legacy_special_token_ids": true, |
| "hidden_size": 128, |
| "intermediate_size": 384, |
| "moe_intermediate_size": 192, |
| "num_hidden_layers": 8, |
| "num_attention_heads": 4, |
| "num_key_value_heads": 1, |
| "num_global_key_value_heads": 1, |
| "head_dim": 32, |
| "global_head_dim": 32, |
| "num_experts": 4, |
| "top_k_experts": 2, |
| "sliding_window": 64, |
| "max_position_embeddings": 1024, |
| "layer_pattern": "sssssFsF", |
| "canvas_length": 8, |
| "max_parameters": 5000000, |
| "vision_hidden_size": 32, |
| "vision_intermediate_size": 64, |
| "vision_num_hidden_layers": 1, |
| "vision_num_attention_heads": 4, |
| "vision_head_dim": 8, |
| "vision_position_embedding_size": 64, |
| "block_size": 64, |
| "batch_size": 8, |
| "gradient_accumulation_steps": 2, |
| "ar_steps": 0, |
| "partial_diffusion_steps": 0, |
| "full_diffusion_steps": 140000, |
| "ar_learning_rate": 0.0002, |
| "diffusion_learning_rate": 7.41e-05, |
| "minimum_learning_rate": 1e-05, |
| "ar_warmup_steps": 1000, |
| "partial_warmup_steps": 500, |
| "full_warmup_steps": 0, |
| "adam_beta1": 0.9, |
| "adam_beta2": 0.95, |
| "adam_eps": 1e-08, |
| "weight_decay": 0.01, |
| "grad_clip": 1.0, |
| "error_on_nonfinite_gradients": true, |
| "first_pass_loss_weight": 1.0, |
| "clean_position_loss_weight": 1.0, |
| "partial_ar_aux_weight": 1.0, |
| "full_ar_aux_weight": 1.0, |
| "partial_self_conditioning_probability": 1.0, |
| "full_self_conditioning_probability": 1.0, |
| "separate_noise_for_second_pass": false, |
| "ema_decay": 0.999, |
| "min_corruption_rate": 0.0, |
| "max_corruption_rate": 1.0, |
| "eval_stochastic_corruption": true, |
| "log_steps": 50, |
| "eval_steps": 1000, |
| "eval_batches": 8, |
| "save_steps": 10000, |
| "seed": 1234, |
| "device": "cuda", |
| "dtype": "float32", |
| "num_workers": 0, |
| "generate_prompts": [ |
| "Once upon", |
| "There was a little", |
| "One day" |
| ], |
| "generate_new_tokens": 64, |
| "max_denoising_steps": 64, |
| "entropy_bound": 1.0, |
| "t_min": 0.4, |
| "t_max": 0.8, |
| "stability_threshold": 3, |
| "confidence_threshold": 0.05 |
| }, |
| "config": { |
| "transformers_version": "5.14.1", |
| "architectures": [ |
| "DiffusionGemmaForBlockDiffusion" |
| ], |
| "output_hidden_states": false, |
| "return_dict": true, |
| "dtype": "float32", |
| "chunk_size_feed_forward": 0, |
| "is_encoder_decoder": false, |
| "id2label": { |
| "0": "LABEL_0", |
| "1": "LABEL_1" |
| }, |
| "label2id": { |
| "LABEL_0": 0, |
| "LABEL_1": 1 |
| }, |
| "problem_type": null, |
| "text_config": { |
| "architectures": null, |
| "output_hidden_states": false, |
| "return_dict": true, |
| "dtype": "float32", |
| "chunk_size_feed_forward": 0, |
| "is_encoder_decoder": false, |
| "id2label": { |
| "0": "LABEL_0", |
| "1": "LABEL_1" |
| }, |
| "label2id": { |
| "LABEL_0": 0, |
| "LABEL_1": 1 |
| }, |
| "problem_type": null, |
| "vocab_size": 1003, |
| "hidden_size": 128, |
| "intermediate_size": 384, |
| "num_hidden_layers": 8, |
| "num_attention_heads": 4, |
| "num_key_value_heads": 1, |
| "head_dim": 32, |
| "hidden_activation": "gelu_pytorch_tanh", |
| "max_position_embeddings": 1024, |
| "initializer_range": 0.02, |
| "rms_norm_eps": 1e-06, |
| "pad_token_id": 1000, |
| "eos_token_id": 1001, |
| "bos_token_id": 1000, |
| "tie_word_embeddings": true, |
| "rope_parameters": { |
| "full_attention": { |
| "partial_rotary_factor": 0.25, |
| "rope_theta": 1000000.0, |
| "rope_type": "proportional" |
| }, |
| "sliding_attention": { |
| "rope_theta": 10000.0, |
| "rope_type": "default" |
| } |
| }, |
| "attention_bias": false, |
| "attention_dropout": 0.0, |
| "sliding_window": 64, |
| "layer_types": [ |
| "sliding_attention", |
| "sliding_attention", |
| "sliding_attention", |
| "sliding_attention", |
| "sliding_attention", |
| "full_attention", |
| "sliding_attention", |
| "full_attention" |
| ], |
| "use_bidirectional_attention": "vision", |
| "num_global_key_value_heads": 1, |
| "global_head_dim": 32, |
| "num_experts": 4, |
| "top_k_experts": 2, |
| "moe_intermediate_size": 192, |
| "_name_or_path": "", |
| "model_type": "diffusion_gemma_text", |
| "output_attentions": false |
| }, |
| "vision_config": { |
| "architectures": null, |
| "output_hidden_states": false, |
| "return_dict": true, |
| "dtype": "float32", |
| "chunk_size_feed_forward": 0, |
| "is_encoder_decoder": false, |
| "id2label": { |
| "0": "LABEL_0", |
| "1": "LABEL_1" |
| }, |
| "label2id": { |
| "LABEL_0": 0, |
| "LABEL_1": 1 |
| }, |
| "problem_type": null, |
| "hidden_size": 32, |
| "intermediate_size": 64, |
| "num_hidden_layers": 1, |
| "num_attention_heads": 4, |
| "num_key_value_heads": 4, |
| "head_dim": 8, |
| "hidden_activation": "gelu_pytorch_tanh", |
| "rms_norm_eps": 1e-06, |
| "max_position_embeddings": 256, |
| "attention_bias": false, |
| "attention_dropout": 0.0, |
| "rope_parameters": { |
| "rope_theta": 100.0, |
| "rope_type": "default" |
| }, |
| "pooling_kernel_size": 1, |
| "patch_size": 16, |
| "position_embedding_size": 64, |
| "use_clipped_linears": false, |
| "standardize": false, |
| "initializer_range": 0.02, |
| "_name_or_path": "", |
| "model_type": "gemma4_vision", |
| "output_attentions": false |
| }, |
| "boi_token_id": 255999, |
| "eoi_token_id": 258882, |
| "image_token_id": 258880, |
| "initializer_range": 0.02, |
| "tie_word_embeddings": true, |
| "canvas_length": 8, |
| "_name_or_path": "/home/codex/tinyModels/runs/diffusion_gemma_100p_scratch_full220k/checkpoints/full-0080000/hf", |
| "model_type": "diffusion_gemma", |
| "output_attentions": false |
| }, |
| "generation_config": { |
| "max_new_tokens": 64, |
| "max_length": null, |
| "return_dict_in_generate": true, |
| "max_denoising_steps": 64, |
| "sampler_config": { |
| "entropy_bound": 1.0, |
| "_cls_name": "EntropyBoundSamplerConfig" |
| }, |
| "t_min": 0.4, |
| "t_max": 0.8, |
| "stability_threshold": 3, |
| "confidence_threshold": 0.05, |
| "cache_implementation": "dynamic", |
| "cache_config": null, |
| "disable_compile": null, |
| "bos_token_id": 1000, |
| "pad_token_id": 1000, |
| "eos_token_id": 1001, |
| "_from_model_config": null, |
| "transformers_version": "5.14.1" |
| } |
| } |
