Buckets:
| { | |
| "model": { | |
| "d_model": 384, | |
| "vocab_size": 16384, | |
| "n_layers": 18, | |
| "block": { | |
| "sequence_mixer": { | |
| "name": "default", | |
| "n_heads": 8, | |
| "n_kv_heads": 2, | |
| "head_dim": 32, | |
| "bias": false, | |
| "gate": { | |
| "granularity": "headwise", | |
| "full_precision": true, | |
| "_CLASS_": "lobotomy.modeling.nn.attention.GateConfig" | |
| }, | |
| "rope": { | |
| "name": "default", | |
| "theta": 10000, | |
| "full_precision": true, | |
| "no_global_rope": false, | |
| "hope": { | |
| "training_sequence_length": 2048, | |
| "_CLASS_": "lobotomy.modeling.nn.rope.HoPEConfig" | |
| }, | |
| "partial_rotary_factor": 1.0, | |
| "_CLASS_": "lobotomy.modeling.nn.rope.RoPEConfig" | |
| }, | |
| "qk_norm": { | |
| "name": "rms", | |
| "eps": 1e-06, | |
| "bias": false, | |
| "dtype": "bfloat16", | |
| "_CLASS_": "lobotomy.modeling.nn.layer_norm.LayerNormConfig" | |
| }, | |
| "backend": "flex", | |
| "dtype": "bfloat16", | |
| "sliding_window": { | |
| "pattern": [ | |
| 128, | |
| 128, | |
| -1 | |
| ], | |
| "force_full_attention_on_first_layer": false, | |
| "force_full_attention_on_last_layer": true, | |
| "_CLASS_": "lobotomy.modeling.nn.attention.SlidingWindowAttentionConfig" | |
| }, | |
| "use_head_qk_norm": false, | |
| "use_attention_sink": true, | |
| "_CLASS_": "lobotomy.modeling.nn.attention.AttentionConfig", | |
| "type": "attention" | |
| }, | |
| "layer_norm": { | |
| "name": "rms", | |
| "eps": 1e-06, | |
| "bias": false, | |
| "dtype": "bfloat16", | |
| "_CLASS_": "lobotomy.modeling.nn.layer_norm.LayerNormConfig" | |
| }, | |
| "feed_forward": { | |
| "hidden_size": 2560, | |
| "name": "default", | |
| "bias": false, | |
| "dtype": "bfloat16", | |
| "activation": "silu", | |
| "_CLASS_": "lobotomy.modeling.nn.feed_forward.FeedForwardConfig" | |
| }, | |
| "name": "default", | |
| "_CLASS_": "lobotomy.modeling.nn.transformer.config.TransformerBlockConfig" | |
| }, | |
| "lm_head": { | |
| "name": "default", | |
| "layer_norm": { | |
| "name": "rms", | |
| "eps": 1e-06, | |
| "bias": false, | |
| "dtype": "bfloat16", | |
| "_CLASS_": "lobotomy.modeling.nn.layer_norm.LayerNormConfig" | |
| }, | |
| "bias": false, | |
| "dtype": "bfloat16", | |
| "loss_implementation": "fused_linear", | |
| "_CLASS_": "lobotomy.modeling.nn.lm_head.LMHeadConfig" | |
| }, | |
| "name": "default", | |
| "dtype": "bfloat16", | |
| "residual_dtype": "bfloat16", | |
| "init_method": "normal", | |
| "init_seed": 0, | |
| "init_std": 0.02, | |
| "tie_word_embeddings": true, | |
| "_CLASS_": "lobotomy.modeling.nn.transformer.config.TransformerConfig" | |
| }, | |
| "dataset": { | |
| "tokenizer": { | |
| "vocab_size": 16384, | |
| "eos_token_id": 16001, | |
| "pad_token_id": 16002, | |
| "identifier": "glossia/Rambley-Tokenizer-v1-16k", | |
| "backend": "hf", | |
| "special_tokens": {}, | |
| "_CLASS_": "lobotomy.modeling.tokenizer.TokenizerConfig" | |
| } | |
| } | |
| } |
Xet Storage Details
- Size:
- 3.08 kB
- Xet hash:
- 3d9d5d53c5c48625449d0d800be3a6d352827f35f7a2cea7fec8eda235bc8512
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.