| { | |
| "model_type": "foresightloop", | |
| "architectures": [ | |
| "ForesightLoop" | |
| ], | |
| "checkpoint_format": "foresightloop-safetensors-v1", | |
| "backbone_config": { | |
| "vocab": 151936, | |
| "dim": 1024, | |
| "layers": 28, | |
| "heads": 16, | |
| "kv_heads": 8, | |
| "head_dim": 128, | |
| "intermediate": 3072, | |
| "window": 64, | |
| "eps": 1e-06, | |
| "theta": 1000000.0 | |
| }, | |
| "num_loops": 8, | |
| "residual_lambda": 0.5, | |
| "cross_loop_aggregation": "attnres", | |
| "source_rmsnorm": true, | |
| "cross_loop_eps": 1e-06, | |
| "cross_loop_source": "previous_token", | |
| "source_trajectory_offset": null, | |
| "torch_dtype": "float32", | |
| "trained_tokens": 6000082944, | |
| "training_steps": 45777, | |
| "tokenizer_repo": "Qwen/Qwen3-0.6B", | |
| "tokenizer_revision": "c1899de289a04d12100db370d81485cdf75e47ca" | |
| } | |