| { |
| "model_type": "loopvl", |
| "architectures": [ |
| "LoopVLForConditionalGeneration" |
| ], |
| "format_version": 2, |
| "loopvl_config": { |
| "vision": { |
| "model_path": ".", |
| "hidden_size": 1024, |
| "patch_size": 14, |
| "input_size": 336, |
| "grid_height": 24, |
| "grid_width": 24, |
| "visual_tokens": 576, |
| "dynamic_resolution": true, |
| "min_visual_tokens": 256, |
| "max_visual_tokens": 2048, |
| "attn_implementation": "sdpa" |
| }, |
| "language": { |
| "model_path": ".", |
| "hidden_size": 1536, |
| "num_attention_heads": 12, |
| "head_dim": 128, |
| "H_cycles": 2, |
| "L_cycles": 3, |
| "rope_theta": 10000, |
| "rope_parameters": { |
| "rope_type": "yarn", |
| "rope_theta": 10000, |
| "factor": 2, |
| "original_max_position_embeddings": 4096, |
| "beta_fast": 32, |
| "beta_slow": 1 |
| }, |
| "max_position_embeddings": 8192, |
| "attn_implementation": "sdpa" |
| }, |
| "recurrent_visual": { |
| "rope_mode": "hybrid_2d", |
| "use_visual_anchor": true, |
| "use_visual_gate": true, |
| "gate_dim": 256, |
| "anchor_raw_init": -2, |
| "anchor_max_scale": 0.5, |
| "gate_saturation_metric_lower": 0.05, |
| "gate_saturation_metric_upper": 1.95 |
| }, |
| "inference": { |
| "max_seq_len": 8192, |
| "use_cache": false |
| } |
| }, |
| "text_config": { |
| "H_cycles": 2, |
| "L_cycles": 3, |
| "architectures": [ |
| "HrmTextForCausalLM" |
| ], |
| "attention_bias": false, |
| "attention_dropout": 0, |
| "bos_token_id": 6, |
| "dtype": "bfloat16", |
| "embedding_scale": 39.191835884530846, |
| "eos_token_id": 11, |
| "head_dim": 128, |
| "hidden_act": "silu", |
| "hidden_size": 1536, |
| "initializer_range": 0.025515518153991442, |
| "intermediate_size": 4096, |
| "max_position_embeddings": 8192, |
| "mlp_bias": false, |
| "model_type": "hrm_text", |
| "num_attention_heads": 12, |
| "num_hidden_layers": 128, |
| "num_key_value_heads": 12, |
| "num_layers_per_stack": 16, |
| "pad_token_id": 5, |
| "prefix_lm": true, |
| "rms_norm_eps": 0.000001, |
| "rope_parameters": { |
| "beta_fast": 32, |
| "beta_slow": 1, |
| "factor": 2, |
| "original_max_position_embeddings": 4096, |
| "rope_theta": 10000, |
| "rope_type": "yarn" |
| }, |
| "rope_theta": 10000, |
| "tie_word_embeddings": false, |
| "transformers_version": "5.15.1", |
| "use_cache": false, |
| "vocab_size": 65536 |
| }, |
| "vision_config": { |
| "architectures": [ |
| "PenguinVLVisionEncoderModel" |
| ], |
| "attention_bias": false, |
| "attention_dropout": 0, |
| "bos_token_id": 151643, |
| "eos_token_id": 151645, |
| "head_dim": 128, |
| "hidden_act": "silu", |
| "hidden_size": 1024, |
| "initializer_range": 0.02, |
| "intermediate_size": 3072, |
| "layer_norm_eps": 0.000001, |
| "max_position_embeddings": 40960, |
| "max_window_layers": 28, |
| "model_type": "penguinvl_vision_encoder", |
| "num_attention_heads": 16, |
| "num_channels": 3, |
| "num_hidden_layers": 28, |
| "num_key_value_heads": 8, |
| "patch_size": 14, |
| "rms_norm_eps": 0.000001, |
| "rope_scaling": null, |
| "rope_theta": 1000000, |
| "sliding_window": null, |
| "tie_word_embeddings": true, |
| "torch_dtype": "bfloat16", |
| "transformers_version": "4.51.3", |
| "use_cache": true, |
| "use_sliding_window": false, |
| "vocab_size": 151936 |
| }, |
| "component_dtypes": { |
| "vision": "bfloat16", |
| "language": "bfloat16", |
| "projector": "float32", |
| "visual_gate": "float32", |
| "visual_anchor": "float32" |
| }, |
| "loader": "modeling_loopvl.LoopVLForConditionalGeneration.from_pretrained", |
| "note": "Single-file weights and configuration only. Use the runtime/ loader from the LoopVL GitHub repository.", |
| "code_repository": "https://github.com/Tier-Flow/LoopVL" |
| } |
|
|
