Hugging Face's logo

embrelgrim
/
grimoire

Text Generation
Russian
tool-use
function-calling
recurrent-depth
qwen2
russian
Model card Files
xet
 Community
grimoire
File size: 3,734 Bytes
0d03a36
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
{
  "config": {
    "mode": "train",
    "preset": "qwen36m",
    "data_dir": "data_final3",
    "train_file": "",
    "val_file": "",
    "vocab_data_dir": "",
    "max_samples": 0,
    "max_tokens": 40000000,
    "val_samples": 1000,
    "val_batches": 40,
    "allow_tiny_data": false,
    "split_tool_segments": true,
    "init": "qwen",
    "model_name": "auto",
    "tokenizer_name": "",
    "base_layer": 6,
    "base_layers": 1,
    "d_model": 896,
    "n_heads": 14,
    "d_head": 64,
    "kv_heads": 2,
    "intermediate_size": 4864,
    "attention_bias": true,
    "rope_theta": 1000000.0,
    "rms_eps": 1e-06,
    "depth": 8,
    "gen_mlp": 128,
    "factory_prefix": 0,
    "max_seq_len": 768,
    "vocab_size": 11264,
    "vocab_strategy": "coverage",
    "coverage": 0.999,
    "merge_weights": "mean",
    "vocab_pack": "runs/night_glm9q/compact_vocab.pt",
    "init_ckpt": "runs/night_glm9q/checkpoints/best.pt",
    "init_skip_embeddings": false,
    "init_extend_vocab": false,
    "seed": 42,
    "batch_size": 4,
    "grad_accum": 4,
    "lr": 4e-05,
    "lr_base": 1e-05,
    "weight_decay": 0.01,
    "warmup": 200,
    "grad_clip": 1.0,
    "min_lr_ratio": 0.1,
    "hours": 1.0,
    "extra_hours": 0.0,
    "max_steps": 0,
    "mask": "assistant",
    "amp": "auto",
    "device": "auto",
    "out": "runs/assistant8",
    "resume": "",
    "run": "",
    "student_run": "",
    "teacher": "Qwen/Qwen2.5-1.5B-Instruct",
    "teacher_data_teacher": "Qwen/Qwen2.5-3B-Instruct",
    "teacher_data_backend": "transformers",
    "teacher_data_url": "http://127.0.0.1:8080/v1/chat/completions",
    "teacher_data_parallel": 1,
    "teacher_data_samples": 128,
    "teacher_data_max_new": 192,
    "teacher_data_min_accept": 0.8,
    "teacher_data_use_accepted": false,
    "teacher_data_seed": 42,
    "bridge_samples": 128,
    "bridge_steps": 300,
    "bridge_max_new": 96,
    "bridge_batch_size": 2,
    "bridge_lr": 7e-06,
    "bridge_lr_base": 1.5e-06,
    "bridge_save_every": 100,
    "teacher_device": "auto",
    "teacher_dtype": "fp16",
    "df_questions": 64,
    "df_steps": 600,
    "df_max_new": 48,
    "df_rollout_temp": 0.75,
    "df_top_p": 0.92,
    "df_kd_temperature": 1.2,
    "df_hidden_weight": 0.01,
    "df_anchor_dir": "",
    "df_anchor_weight": 0.2,
    "df_lr": 1.5e-06,
    "df_lr_base": 4e-07,
    "df_save_every": 100,
    "eval_seconds": 240,
    "save_minutes": 15,
    "sample_seconds": 900,
    "log_seconds": 10,
    "no_samples": false,
    "demo": true,
    "demo_force": false,
    "demo_max_new": 64,
    "export_dtype": "fp16",
    "compile": false,
    "grad_checkpoint": false,
    "opt": "adamw",
    "muon_lr": 0.02,
    "tools": true,
    "max_new": 192,
    "temperature": 0.0,
    "rep_penalty": 1.15,
    "prompt": "",
    "yes": true,
    "dry_run": false
  },
  "arch": {
    "d_model": 896,
    "attention_bias": true,
    "rope_theta": 1000000.0,
    "rms_eps": 1e-06,
    "n_heads": 14,
    "d_head": 64,
    "kv_heads": 2,
    "intermediate_size": 4864,
    "base_layers": 1,
    "depth": 8,
    "gen_mlp": 128,
    "factory_prefix": 0,
    "max_seq_len": 768,
    "vocab_size": 11264
  },
  "params": {
    "g_init": 896,
    "depth_scale": 8,
    "base_layer": 14912384,
    "norm": 896,
    "embed_tokens": 10092544,
    "morph_patcher": 158464,
    "gen_layer": 1792,
    "factory": 10682368,
    "TOTAL": 35849352
  },
  "vocab_meta": {
    "strategy": "coverage",
    "vocab_size": 11264,
    "coverage": 0.9984156092959803,
    "tokenizer_len": 151665,
    "d_model": 896,
    "n_heads": 14,
    "d_head": 64,
    "merge_weights": "mean",
    "pad_old_id": 151643,
    "created_at": "2026-10-27T17:35:28"
  },
  "created_at": "2026-10-30T16:45:17"
}