yujiepan commited on
Commit
facb9f1
·
verified ·
1 Parent(s): 6963637

Upload folder using huggingface_hub

Browse files
.meta/create_model.py ADDED
@@ -0,0 +1,218 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str:
2
+ import json
3
+ from pathlib import Path
4
+
5
+ import torch
6
+ from huggingface_hub import hf_hub_download
7
+ from transformers import AutoConfig, AutoModel, set_seed
8
+
9
+ folder = Path(save_folder)
10
+ folder.mkdir(parents=True, exist_ok=True)
11
+ for name in (
12
+ "configuration_deepseek_v2.py",
13
+ "modeling_unlimitedocr.py",
14
+ "modeling_deepseekv2.py",
15
+ "_sparse_base_impl.py",
16
+ "dash_attention.py",
17
+ "packing_patch.py",
18
+ "sparse_gather_triton.py",
19
+ "sparse_gather_e2e_triton.py",
20
+ "deepencoder.py",
21
+ "conversation.py",
22
+ "tokenizer.json",
23
+ "tokenizer_config.json",
24
+ "special_tokens_map.json",
25
+ "generation_config.json",
26
+ ):
27
+ (folder / name).write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
28
+ config = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
29
+ # Keep four TP heads of dimension 32 independently from the residual width.
30
+ # Q/K/V project to 128 attention channels; o_proj returns to hidden_size.
31
+ dimensions = dict(
32
+ hidden_size=32,
33
+ head_dim=32,
34
+ intermediate_size=256,
35
+ moe_intermediate_size=64,
36
+ num_hidden_layers=4,
37
+ num_attention_heads=4,
38
+ num_key_value_heads=4,
39
+ n_routed_experts=16,
40
+ v_head_dim=32,
41
+ )
42
+ config.update(dimensions)
43
+ config["language_config"].update(dimensions)
44
+ sparse = config["sparse_attention_config"]
45
+ sparse["sparse_index_dim"] = 32
46
+ for name in ("sparse_attention_freq", "dash_scale_per_layer", "dash_floor_per_layer"):
47
+ sparse[name] = sparse[name][:4]
48
+ vision = config["vision_config"]["width"]
49
+ # Two complete local/local/global SAM cycles; both encoder families remain.
50
+ vision["sam_vit_b"].update(
51
+ width=32, heads=4, layers=6, global_attn_indexes=[2, 5], neck_channels=8, downsample_channels=[16, 32]
52
+ )
53
+ vision["clip-l-14-224"].update(width=32, heads=4, layers=4)
54
+ config["projector_config"].update(input_dim=64, n_embed=config["hidden_size"])
55
+
56
+ def replace_once(name, old, new):
57
+ path = folder / name
58
+ code = path.read_text()
59
+ if code.count(old) != 1:
60
+ raise ValueError(f"Unexpected upstream implementation in {name}: {old}")
61
+ path.write_text(code.replace(old, new, 1))
62
+
63
+ # Transformers' LlamaAttention already supports an explicit head_dim. DASH
64
+ # must use that same dimension for its attention scaling and Triton kernel.
65
+ replace_once(
66
+ "dash_attention.py",
67
+ "self.head_dim = config.hidden_size // config.num_attention_heads",
68
+ 'self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)',
69
+ )
70
+
71
+ # The reference hardcodes encoder widths; expose its existing builders to
72
+ # config without replacing attention, routing, position encodings or APIs.
73
+ replace_once(
74
+ "modeling_unlimitedocr.py",
75
+ "self.sam_model = build_sam_vit_b()",
76
+ "self.sam_model = build_sam_vit_b(config=config)",
77
+ )
78
+ replace_once(
79
+ "modeling_unlimitedocr.py",
80
+ "self.vision_model = build_clip_l()",
81
+ "self.vision_model = build_clip_l(config=config)",
82
+ )
83
+ replace_once("modeling_unlimitedocr.py", " n_embed = 1280", " n_embed = config.hidden_size")
84
+ replace_once(
85
+ "modeling_unlimitedocr.py",
86
+ "input_dim=2048, n_embed=n_embed",
87
+ 'input_dim=config.projector_config["input_dim"], n_embed=n_embed',
88
+ )
89
+ replace_once(
90
+ "deepencoder.py",
91
+ "def build_clip_l():",
92
+ """def build_clip_l(config=None):
93
+ cfg = copy.deepcopy(vit_model_cfg)
94
+ if config is not None:
95
+ spec = config.vision_config['width']['clip-l-14-224']
96
+ cfg.update(num_layers=spec['layers'], hidden_size=spec['width'],
97
+ num_heads=spec['heads'], num_attention_heads=spec['heads'],
98
+ ffn_hidden_size=spec['width'] * 4)""",
99
+ )
100
+ replace_once("deepencoder.py", "cfg=vit_model_cfg,", "cfg=cfg,")
101
+ replace_once(
102
+ "deepencoder.py",
103
+ "def build_sam_vit_b(checkpoint=None):",
104
+ """def build_sam_vit_b(checkpoint=None, config=None):
105
+ if config is not None:
106
+ spec = config.vision_config['width']['sam_vit_b']
107
+ encoder = ImageEncoderViT(embed_dim=spec['width'], depth=spec['layers'],
108
+ num_heads=spec['heads'], global_attn_indexes=spec['global_attn_indexes'],
109
+ out_chans=spec['neck_channels'], window_size=14, use_rel_pos=True,
110
+ norm_layer=partial(torch.nn.LayerNorm, eps=1e-6))
111
+ c1, c2 = spec['downsample_channels']
112
+ encoder.net_2 = nn.Conv2d(spec['neck_channels'], c1, 3, stride=2, padding=1, bias=False)
113
+ encoder.net_3 = nn.Conv2d(c1, c2, 3, stride=2, padding=1, bias=False)
114
+ return encoder""",
115
+ )
116
+ (folder / "config.json").write_text(json.dumps(config, indent=2) + "\n")
117
+ set_seed(42)
118
+ # This pinned reference registers only mha_eager. Its infer() explicitly
119
+ # runs DASH sparse prefill/decode; the vision encoders use SDPA.
120
+ model = AutoModel.from_config(
121
+ AutoConfig.from_pretrained(folder, trust_remote_code=True), trust_remote_code=True, torch_dtype=torch.bfloat16
122
+ ).to(torch.bfloat16)
123
+ model.save_pretrained(folder)
124
+ (folder / "config.json").write_text(json.dumps(config, indent=2) + "\n")
125
+ (folder / "generation_config.json").write_bytes(
126
+ Path(hf_hub_download(source_model_id, "generation_config.json", revision=source_revision)).read_bytes()
127
+ )
128
+ # SGLang's official wheel also hardcodes the full-size vision constructors.
129
+ # Its external-model hook changes only construction; DASH stays upstream.
130
+ runtime = folder / "unlimited_ocr_runtime"
131
+ runtime.mkdir(exist_ok=True)
132
+ (runtime / "__init__.py").write_text("")
133
+ (runtime / "model.py").write_text("""from copy import deepcopy
134
+ from functools import partial
135
+ from unittest.mock import patch
136
+
137
+ import torch
138
+ from torch import nn
139
+ from sglang.srt.models import deepseek_ocr as vision
140
+ from sglang.srt.models import unlimited_ocr as base
141
+ from sglang.srt.models import unlimited_ocr_v2 as dash
142
+ from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
143
+ from sglang.srt.models.unlimited_ocr_v2 import UnlimitedOCRV2ForCausalLM as UpstreamModel
144
+
145
+
146
+ class ConfigAttention(dash.UnlimitedOCRV2Attention):
147
+ def __init__(self, config, layer_id, quant_config=None, prefix=""):
148
+ # Use the pinned constructor and forward, with a separate residual width.
149
+ attention_config = deepcopy(config)
150
+ attention_config.hidden_size = config.num_attention_heads * config.head_dim
151
+ super().__init__(attention_config, layer_id, quant_config, prefix)
152
+ self.hidden_size = config.hidden_size
153
+ outputs = {
154
+ "q_proj": config.num_attention_heads * self.head_dim,
155
+ "k_proj": config.num_key_value_heads * self.head_dim,
156
+ "v_proj": config.num_key_value_heads * self.head_dim,
157
+ "index_q_proj": self.idx_head_dim,
158
+ "index_k_proj": self.idx_head_dim,
159
+ }
160
+ for name, output_size in outputs.items():
161
+ setattr(self, name, ColumnParallelLinear(config.hidden_size, output_size,
162
+ bias=False, gather_output=name.startswith("index_"), quant_config=quant_config,
163
+ prefix=f"{prefix}.{name}" if prefix else name))
164
+ self.o_proj = RowParallelLinear(config.num_attention_heads * self.head_dim,
165
+ config.hidden_size, bias=False, quant_config=quant_config,
166
+ prefix=f"{prefix}.o_proj" if prefix else "o_proj")
167
+
168
+
169
+ class UnlimitedOCRV2ForCausalLM(UpstreamModel):
170
+ def __init__(self, *, config, quant_config=None, prefix=""):
171
+ specs = config.vision_config.width
172
+
173
+ def sam():
174
+ spec = specs["sam_vit_b"]
175
+ encoder = vision.ImageEncoderViT(embed_dim=spec["width"], depth=spec["layers"],
176
+ num_heads=spec["heads"], global_attn_indexes=spec["global_attn_indexes"],
177
+ out_chans=spec["neck_channels"], window_size=14, use_rel_pos=True,
178
+ norm_layer=partial(nn.LayerNorm, eps=1e-6))
179
+ c1, c2 = spec["downsample_channels"]
180
+ encoder.net_2 = nn.Conv2d(spec["neck_channels"], c1, 3, stride=2, padding=1, bias=False)
181
+ encoder.net_3 = nn.Conv2d(c1, c2, 3, stride=2, padding=1, bias=False)
182
+ return encoder
183
+
184
+ def clip():
185
+ spec = specs["clip-l-14-224"]
186
+ cfg = deepcopy(vision.vit_model_cfg)
187
+ cfg.update(num_layers=spec["layers"], hidden_size=spec["width"], num_heads=spec["heads"],
188
+ num_attention_heads=spec["heads"], ffn_hidden_size=spec["width"] * 4)
189
+ return vision.VitModel(cfg)
190
+
191
+ with (patch.object(base, "build_sam_vit_b", sam),
192
+ patch.object(base, "build_clip_l", clip),
193
+ patch.object(dash, "UnlimitedOCRV2Attention", ConfigAttention)):
194
+ super().__init__(config=config, quant_config=quant_config, prefix=prefix)
195
+
196
+
197
+ EntryClass = UnlimitedOCRV2ForCausalLM
198
+ """)
199
+ (folder / "pyproject.toml").write_text("""[build-system]
200
+ requires = ["setuptools>=61"]
201
+ build-backend = "setuptools.build_meta"
202
+
203
+ [project]
204
+ name = "unlimited-ocr-config-runtime"
205
+ version = "0.1.0"
206
+ requires-python = ">=3.10"
207
+
208
+ [tool.setuptools.packages.find]
209
+ include = ["unlimited_ocr_runtime*"]
210
+ """)
211
+ return repr(model)
212
+
213
+
214
+ create_model(
215
+ source_model_id="baidu/Unlimited-OCR-2",
216
+ source_revision="c14071b549f4fcab10909f3c4acc6a2edb366947",
217
+ save_folder="/tmp/unlimited-ocr-2",
218
+ )
.meta/env.json ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "source_model": {
3
+ "name": "baidu/Unlimited-OCR-2",
4
+ "commit_id": "c14071b549f4fcab10909f3c4acc6a2edb366947"
5
+ },
6
+ "environments": {
7
+ "creation": {
8
+ "python": "3.12.13",
9
+ "packages": {
10
+ "addict": "2.4.0",
11
+ "easydict": "1.13",
12
+ "einops": "0.9.0.dev0",
13
+ "huggingface_hub": "0.36.0",
14
+ "kernels": "0.7.0",
15
+ "safetensors": "0.8.0",
16
+ "tokenizers": "0.21.4",
17
+ "torch": "2.13.0+cu126",
18
+ "torchvision": "0.28.0+cu126",
19
+ "transformers": "4.52.1"
20
+ },
21
+ "cuda": "12.6",
22
+ "hardware": {
23
+ "gpu_model": "NVIDIA H20"
24
+ }
25
+ },
26
+ "sglang": {
27
+ "python": "3.12.13",
28
+ "packages": {
29
+ "apache-tvm-ffi": "0.1.11",
30
+ "flashinfer-python": "0.6.18.post1",
31
+ "huggingface_hub": "1.9.0",
32
+ "kernels": "0.14.1",
33
+ "ninja": "1.13.0",
34
+ "safetensors": "0.8.0",
35
+ "sgl-deep-gemm": "0.1.4.post1",
36
+ "sglang": "0.5.20.dev13+g75b9b06b02",
37
+ "sglang-kernel": "0.4.7",
38
+ "tokenizers": "0.22.2",
39
+ "torch": "2.13.0+cu126",
40
+ "transformers": "5.12.1",
41
+ "triton": "3.7.1"
42
+ },
43
+ "cuda": "12.6",
44
+ "hardware": {
45
+ "gpu_model": "NVIDIA H20"
46
+ }
47
+ },
48
+ "transformers": {
49
+ "python": "3.12.13",
50
+ "packages": {
51
+ "Pillow": "12.3.0",
52
+ "addict": "2.4.0",
53
+ "easydict": "1.13",
54
+ "einops": "0.9.0.dev0",
55
+ "huggingface_hub": "0.36.0",
56
+ "kernels": "0.7.0",
57
+ "safetensors": "0.8.0",
58
+ "tokenizers": "0.21.4",
59
+ "torch": "2.13.0+cu126",
60
+ "torchvision": "0.28.0+cu126",
61
+ "transformers": "4.52.1",
62
+ "triton": "3.7.1"
63
+ },
64
+ "cuda": "12.6",
65
+ "hardware": {
66
+ "gpu_model": "NVIDIA H20"
67
+ }
68
+ }
69
+ }
70
+ }
.meta/model.log ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ UnlimitedOCRForCausalLM(
2
+ (model): UnlimitedOCRModel(
3
+ (embed_tokens): Embedding(129280, 32, padding_idx=2)
4
+ (layers): ModuleList(
5
+ (0): DeepseekV2DecoderLayer(
6
+ (self_attn): SlidingWindowLlamaAttention(
7
+ (q_proj): Linear(in_features=32, out_features=128, bias=False)
8
+ (k_proj): Linear(in_features=32, out_features=128, bias=False)
9
+ (v_proj): Linear(in_features=32, out_features=128, bias=False)
10
+ (o_proj): Linear(in_features=128, out_features=32, bias=False)
11
+ (rotary_emb): LlamaRotaryEmbedding()
12
+ (indexer): DeepseekV2Indexer(
13
+ (q_proj): Linear(in_features=32, out_features=32, bias=False)
14
+ (k_proj): Linear(in_features=32, out_features=32, bias=False)
15
+ (q_norm): _GemmaRMSNorm()
16
+ (k_norm): _GemmaRMSNorm()
17
+ (rotary_emb_1d): DeepseekV2IndexerRotaryEmbedding()
18
+ (rotary_emb_2d): DeepseekV2IndexerRotary2D()
19
+ (rotary_emb): DeepseekV2IndexerRotary2D()
20
+ )
21
+ (rotary_emb_2d_main): DeepseekV2IndexerRotary2D()
22
+ (dash_router): DashAttention(
23
+ (q_proj): Linear(in_features=32, out_features=32, bias=False)
24
+ (k_proj): Linear(in_features=32, out_features=32, bias=False)
25
+ (q_norm): _RMSNorm()
26
+ (k_norm): _RMSNorm()
27
+ )
28
+ )
29
+ (mlp): DeepseekV2MLP(
30
+ (gate_proj): Linear(in_features=32, out_features=256, bias=False)
31
+ (up_proj): Linear(in_features=32, out_features=256, bias=False)
32
+ (down_proj): Linear(in_features=256, out_features=32, bias=False)
33
+ (act_fn): SiLU()
34
+ )
35
+ (input_layernorm): DeepseekV2RMSNorm()
36
+ (post_attention_layernorm): DeepseekV2RMSNorm()
37
+ )
38
+ (1-3): 3 x DeepseekV2DecoderLayer(
39
+ (self_attn): SlidingWindowLlamaAttention(
40
+ (q_proj): Linear(in_features=32, out_features=128, bias=False)
41
+ (k_proj): Linear(in_features=32, out_features=128, bias=False)
42
+ (v_proj): Linear(in_features=32, out_features=128, bias=False)
43
+ (o_proj): Linear(in_features=128, out_features=32, bias=False)
44
+ (rotary_emb): LlamaRotaryEmbedding()
45
+ (indexer): DeepseekV2Indexer(
46
+ (q_proj): Linear(in_features=32, out_features=32, bias=False)
47
+ (k_proj): Linear(in_features=32, out_features=32, bias=False)
48
+ (q_norm): _GemmaRMSNorm()
49
+ (k_norm): _GemmaRMSNorm()
50
+ (rotary_emb_1d): DeepseekV2IndexerRotaryEmbedding()
51
+ (rotary_emb_2d): DeepseekV2IndexerRotary2D()
52
+ (rotary_emb): DeepseekV2IndexerRotary2D()
53
+ )
54
+ (rotary_emb_2d_main): DeepseekV2IndexerRotary2D()
55
+ (dash_router): DashAttention(
56
+ (q_proj): Linear(in_features=32, out_features=32, bias=False)
57
+ (k_proj): Linear(in_features=32, out_features=32, bias=False)
58
+ (q_norm): _RMSNorm()
59
+ (k_norm): _RMSNorm()
60
+ )
61
+ )
62
+ (mlp): DeepseekV2MoE(
63
+ (experts): ModuleList(
64
+ (0-15): 16 x DeepseekV2MLP(
65
+ (gate_proj): Linear(in_features=32, out_features=64, bias=False)
66
+ (up_proj): Linear(in_features=32, out_features=64, bias=False)
67
+ (down_proj): Linear(in_features=64, out_features=32, bias=False)
68
+ (act_fn): SiLU()
69
+ )
70
+ )
71
+ (gate): MoEGate()
72
+ (shared_experts): DeepseekV2MLP(
73
+ (gate_proj): Linear(in_features=32, out_features=128, bias=False)
74
+ (up_proj): Linear(in_features=32, out_features=128, bias=False)
75
+ (down_proj): Linear(in_features=128, out_features=32, bias=False)
76
+ (act_fn): SiLU()
77
+ )
78
+ )
79
+ (input_layernorm): DeepseekV2RMSNorm()
80
+ (post_attention_layernorm): DeepseekV2RMSNorm()
81
+ )
82
+ )
83
+ (norm): DeepseekV2RMSNorm()
84
+ (sam_model): ImageEncoderViT(
85
+ (patch_embed): PatchEmbed(
86
+ (proj): Conv2d(3, 32, kernel_size=(16, 16), stride=(16, 16))
87
+ )
88
+ (blocks): ModuleList(
89
+ (0-5): 6 x Block(
90
+ (norm1): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
91
+ (attn): Attention(
92
+ (qkv): Linear(in_features=32, out_features=96, bias=True)
93
+ (proj): Linear(in_features=32, out_features=32, bias=True)
94
+ )
95
+ (norm2): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
96
+ (mlp): MLPBlock(
97
+ (lin1): Linear(in_features=32, out_features=128, bias=True)
98
+ (lin2): Linear(in_features=128, out_features=32, bias=True)
99
+ (act): GELU(approximate='none')
100
+ )
101
+ )
102
+ )
103
+ (neck): Sequential(
104
+ (0): Conv2d(32, 8, kernel_size=(1, 1), stride=(1, 1), bias=False)
105
+ (1): LayerNorm2d()
106
+ (2): Conv2d(8, 8, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
107
+ (3): LayerNorm2d()
108
+ )
109
+ (net_2): Conv2d(8, 16, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
110
+ (net_3): Conv2d(16, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
111
+ )
112
+ (vision_model): VitModel(
113
+ (embeddings): CLIPVisionEmbeddings(
114
+ (patch_embedding): Conv2d(3, 32, kernel_size=(14, 14), stride=(14, 14), bias=False)
115
+ (position_embedding): Embedding(257, 32)
116
+ )
117
+ (transformer): NoTPTransformer(
118
+ (layers): ModuleList(
119
+ (0-3): 4 x NoTPTransformerBlock(
120
+ (self_attn): NoTPAttention(
121
+ (qkv_proj): Linear(in_features=32, out_features=96, bias=True)
122
+ (out_proj): Linear(in_features=32, out_features=32, bias=True)
123
+ )
124
+ (mlp): NoTPFeedForward(
125
+ (fc1): Linear(in_features=32, out_features=128, bias=True)
126
+ (fc2): Linear(in_features=128, out_features=32, bias=True)
127
+ )
128
+ (layer_norm1): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
129
+ (layer_norm2): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
130
+ )
131
+ )
132
+ )
133
+ (pre_layrnorm): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
134
+ )
135
+ (projector): MlpProjector(
136
+ (layers): Linear(in_features=64, out_features=32, bias=True)
137
+ )
138
+ )
139
+ (lm_head): Linear(in_features=32, out_features=129280, bias=False)
140
+ )
README.md ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ pipeline_tag: image-text-to-text
4
+ base_model:
5
+ - baidu/Unlimited-OCR-2
6
+ ---
7
+
8
+ This is a randomly initialized tiny model based on [baidu/Unlimited-OCR-2](https://huggingface.co/baidu/Unlimited-OCR-2) for testing and debugging.
9
+
10
+ | Feature | Source | This repo |
11
+ |---|---|---|
12
+ | Checkpoint tensor elements | 3.34B BF16, 257 I64 | 9.04M BF16, 257 I64 |
13
+ | Safetensors weight files | 6.69 GB (2 files) | 18.12 MB (model.safetensors) |
14
+ | Decoder layers / hidden width / dense MLP | 12 / 1,280 / 6,848 | 4 / 32 / 256 |
15
+ | Q / KV heads / head dimension | 10 / 10 / 128 | 4 / 4 / 32 |
16
+ | Routed experts / top-k / shared experts | 64 / 6 / 2 | 16 / 6 / 2 |
17
+ | Expert intermediate width / initial dense layers | 896 / 1 | 64 / 1 |
18
+ | DASH index width / sparse block size | 128 / 32 | 32 / 32 |
19
+ | Page and token RoPE / sparse layer coverage | page2d / all layers | Same |
20
+ | SAM layers / width / global attention layers | 12 / 768 / 2,5,8,11 | 6 / 32 / 2,5 |
21
+ | CLIP layers / width | 24 / 1,024 | 4 / 32 |
22
+ | SAM neck / downsampling channels | 256 / 512,1,024 | 8 / 16,32 |
23
+ | Vision projector input / output | 2,048 / 1,280 | 64 / 32 |
24
+ | Global / crop image size / page visual tokens | 1,024 / 640 / 273 | Same |
25
+ | Vocabulary rows / tied embeddings | 129,280 / no | Same |
26
+ | Serialized precision | BF16 weights + I64 position IDs | Same |
27
+
28
+ Residual width 32 is independent of the four 32-dimensional attention heads: Q/K/V project to 128 channels and the output projection returns to the residual width. The included runtime exposes these config dimensions while preserving the upstream attention kernels.
29
+
30
+ | File path | Size |
31
+ |------|------|
32
+ | model.safetensors | 18.1MB |
33
+
34
+ ### Example usage
35
+
36
+ #### Transformers (✅ verified)
37
+
38
+ Install Transformers 4.52.1, `torchvision`, `Pillow`, `addict`, `easydict` and `einops` and load the included upstream remote code. The source registers only `mha_eager`; its OCR API runs DASH Triton sparse prefill, the reference masked sparse decode, and SDPA in SAM/CLIP.
39
+
40
+ ```python
41
+ def run_transformers(model_id: str):
42
+ import torch
43
+ from PIL import Image, ImageDraw
44
+ from transformers import AutoModel, AutoTokenizer
45
+
46
+ tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
47
+ model = AutoModel.from_pretrained(model_id, trust_remote_code=True, torch_dtype=torch.bfloat16).eval().cuda()
48
+ page = Image.new("RGB", (800, 1000), "white")
49
+ ImageDraw.Draw(page).text((50, 50), "Hello OCR. Page one.", fill="black")
50
+ text = model.infer(tokenizer, prompt="Read this document.", image_file=page, max_new_tokens=10)
51
+ print(text)
52
+ text, tokens = model.infer_multi(tokenizer, image_files=[page, page.copy()], max_new_tokens=10)
53
+ print(text, tokens)
54
+
55
+
56
+ if __name__ == "__main__":
57
+ run_transformers(model_id="tiny-random/unlimited-ocr-2")
58
+ ```
59
+
60
+ #### SGLang (✅ verified)
61
+
62
+ Install Baidu's pinned `sglang-0.5.20.dev13+g75b9b06b02` wheel from the [source repo](https://huggingface.co/baidu/Unlimited-OCR-2/tree/c14071b549f4fcab10909f3c4acc6a2edb366947/wheel). The included external-model package reads the reduced vision config, separates residual width from attention head dimension, and gathers the TP-sharded DASH index projections before normalization. The upstream attention and routing kernels are retained.
63
+
64
+ ```bash
65
+ hf download tiny-random/unlimited-ocr-2 --local-dir ./unlimited-ocr-2
66
+ pip install --no-deps ./unlimited-ocr-2
67
+
68
+ export SGLANG_EXTERNAL_MODEL_PACKAGE=unlimited_ocr_runtime
69
+ export SGLANG_UOCR_EANABLE_DOC_LIMIT=0
70
+ python -m sglang.launch_server \
71
+ --model-path ./unlimited-ocr-2 \
72
+ --tp-size 4 \
73
+ --trust-remote-code \
74
+ --chat-template unlimited-ocr \
75
+ --context-length 4096 \
76
+ --max-total-tokens 8192 \
77
+ --max-running-requests 4 \
78
+ --cuda-graph-backend-decode full \
79
+ --cuda-graph-max-bs-decode 4 \
80
+ --cuda-graph-backend-prefill full \
81
+ --cuda-graph-max-bs-prefill 2048 \
82
+ --cuda-graph-bs-prefill 2048 \
83
+ --max-prefill-tokens 2048 \
84
+ --chunked-prefill-size 2048 \
85
+ --cuda-graph-config '{"prefill": {"backend": "full", "full_prefill_max_req": 4}}' \
86
+ --mem-fraction-static 0.3 \
87
+ --host 0.0.0.0 --port 8000
88
+ ```
89
+
90
+ ```python
91
+ import base64
92
+ from io import BytesIO
93
+
94
+ import requests
95
+ from PIL import Image, ImageDraw
96
+
97
+ page = Image.new("RGB", (800, 1000), "white")
98
+ ImageDraw.Draw(page).text((50, 50), "Hello OCR", fill="black")
99
+ buffer = BytesIO()
100
+ page.save(buffer, format="PNG")
101
+ image = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
102
+ session = requests.Session()
103
+ session.trust_env = False
104
+ response = session.post("http://127.0.0.1:8000/v1/chat/completions", json={
105
+ "model": "./unlimited-ocr-2",
106
+ "messages": [{"role": "user", "content": [
107
+ {"type": "image_url", "image_url": {"url": image}},
108
+ {"type": "text", "text": "<PAGE><image>Read this document."},
109
+ ]}],
110
+ "max_tokens": 10, "temperature": 0,
111
+ }, timeout=120)
112
+ response.raise_for_status()
113
+ print(response.json()["choices"][0]["message"]["content"])
114
+ ```
115
+
116
+ ### Creating this repo
117
+
118
+ - Code: [.meta/create_model.py](.meta/create_model.py)
119
+ - Model structure: [.meta/model.log](.meta/model.log)
120
+ - Environment: [.meta/env.json](.meta/env.json)
121
+
122
+ ### Change log
123
+
124
+ - 2026-10-31: Initial tiny Unlimited-OCR 2 checkpoint with the upstream SR-SWA runtime.
125
+ - 2026-11-01: Separate residual width from the 32-dimensional attention heads to reduce checkpoint size.
126
+
127
+ *This repo is generated by GPT-6.1-Sol.*
_sparse_base_impl.py ADDED
The diff for this file is too large to render. See raw diff
 
config.json ADDED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_ring_window": 128,
3
+ "architectures": [
4
+ "UnlimitedOCRForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "auto_map": {
9
+ "AutoConfig": "modeling_unlimitedocr.UnlimitedOCRConfig",
10
+ "AutoModel": "modeling_unlimitedocr.UnlimitedOCRForCausalLM"
11
+ },
12
+ "aux_loss_alpha": 0.001,
13
+ "bos_token_id": 0,
14
+ "candidate_resolutions": [
15
+ [
16
+ 1024,
17
+ 1024
18
+ ]
19
+ ],
20
+ "eos_token_id": 1,
21
+ "ep_size": 1,
22
+ "first_k_dense_replace": 1,
23
+ "global_view_pos": "head",
24
+ "hidden_act": "silu",
25
+ "hidden_size": 32,
26
+ "initializer_range": 0.02,
27
+ "intermediate_size": 256,
28
+ "kv_lora_rank": null,
29
+ "language_config": {
30
+ "architectures": [
31
+ "DeepseekOCRForCausalLM"
32
+ ],
33
+ "auto_map": {
34
+ "AutoConfig": "configuration_deepseekv2.DeepseekV2Config",
35
+ "AutoModel": "modeling_deepseek.DeepseekV2Model",
36
+ "AutoModelForCausalLM": "modeling_deepseek.DeepseekV2ForCausalLM"
37
+ },
38
+ "bos_token_id": 0,
39
+ "eos_token_id": 1,
40
+ "first_k_dense_replace": 1,
41
+ "hidden_size": 32,
42
+ "intermediate_size": 256,
43
+ "kv_lora_rank": null,
44
+ "lm_head": true,
45
+ "max_position_embeddings": 131072,
46
+ "moe_intermediate_size": 64,
47
+ "n_group": 1,
48
+ "n_routed_experts": 16,
49
+ "n_shared_experts": 2,
50
+ "num_attention_heads": 4,
51
+ "num_experts_per_tok": 6,
52
+ "num_hidden_layers": 4,
53
+ "num_key_value_heads": 4,
54
+ "q_lora_rank": null,
55
+ "qk_nope_head_dim": 0,
56
+ "qk_rope_head_dim": 0,
57
+ "rm_head": false,
58
+ "rope_scaling": {
59
+ "attention_factor": 1.0,
60
+ "beta_fast": 32,
61
+ "beta_slow": 1,
62
+ "factor": 4.0,
63
+ "original_max_position_embeddings": 32768,
64
+ "rope_type": "yarn",
65
+ "type": "yarn"
66
+ },
67
+ "rope_theta": 500000.0,
68
+ "sliding_window_size": 128,
69
+ "topk_group": 1,
70
+ "topk_method": "greedy",
71
+ "torch_dtype": "bfloat16",
72
+ "use_mla": false,
73
+ "v_head_dim": 32,
74
+ "vocab_size": 129280,
75
+ "head_dim": 32
76
+ },
77
+ "lm_head": true,
78
+ "max_position_embeddings": 131072,
79
+ "model_type": "unlimited-ocr",
80
+ "moe_intermediate_size": 64,
81
+ "moe_layer_freq": 1,
82
+ "n_group": 1,
83
+ "n_routed_experts": 16,
84
+ "n_shared_experts": 2,
85
+ "norm_topk_prob": false,
86
+ "num_attention_heads": 4,
87
+ "num_experts_per_tok": 6,
88
+ "num_hidden_layers": 4,
89
+ "num_key_value_heads": 4,
90
+ "pad_token_id": 2,
91
+ "pretraining_tp": 1,
92
+ "projector_config": {
93
+ "input_dim": 64,
94
+ "model_type": "mlp_projector",
95
+ "n_embed": 32,
96
+ "projector_type": "linear"
97
+ },
98
+ "q_lora_rank": null,
99
+ "qk_nope_head_dim": 0,
100
+ "qk_rope_head_dim": 0,
101
+ "rm_head": false,
102
+ "rms_norm_eps": 1e-06,
103
+ "rope_scaling": {
104
+ "attention_factor": 1.0,
105
+ "beta_fast": 32,
106
+ "beta_slow": 1,
107
+ "factor": 4.0,
108
+ "original_max_position_embeddings": 32768,
109
+ "rope_type": "yarn",
110
+ "type": "yarn"
111
+ },
112
+ "rope_theta": 500000.0,
113
+ "routed_scaling_factor": 1.0,
114
+ "scoring_func": "softmax",
115
+ "seq_aux": true,
116
+ "sliding_window": null,
117
+ "sliding_window_size": 128,
118
+ "sparse_attention_config": {
119
+ "dash_floor_per_layer": [
120
+ 0.0001,
121
+ 0.0001,
122
+ 0.0001,
123
+ 0.0001
124
+ ],
125
+ "dash_image_only": true,
126
+ "dash_prior_floor": 0.0001,
127
+ "dash_query_chunk": 2048,
128
+ "dash_router": "entmax_topk_perlayer",
129
+ "dash_scale_per_layer": [
130
+ 128.0,
131
+ 128.0,
132
+ 128.0,
133
+ 128.0
134
+ ],
135
+ "dash_scaling_factor": 128.0,
136
+ "dash_sigma": 1.0,
137
+ "dash_stage0": "head_cls",
138
+ "dash_topk_blocks": 12,
139
+ "dash_use_triton": true,
140
+ "image_bucket_constraint": true,
141
+ "image_token_id": 128815,
142
+ "kl_chunk_size": 1024,
143
+ "kl_log_interval": 1,
144
+ "kl_loss_weight": 1.0,
145
+ "page_token_id": 128827,
146
+ "per_page_image_tokens": 273,
147
+ "sparse_attention_freq": [
148
+ 1,
149
+ 1,
150
+ 1,
151
+ 1
152
+ ],
153
+ "sparse_block_size": 32,
154
+ "sparse_index_dim": 32,
155
+ "sparse_init_block": 1,
156
+ "sparse_local_block": 4,
157
+ "sparse_main_rope_mode": "page2d",
158
+ "sparse_main_rope_tok_freq_mode": "resample",
159
+ "sparse_num_index_heads": 1,
160
+ "sparse_rope_mode": "page2d",
161
+ "sparse_rope_page_mod": 2,
162
+ "sparse_rope_page_pairs": 1,
163
+ "sparse_rope_page_span": 0,
164
+ "sparse_rope_page_theta": 256.0,
165
+ "sparse_rope_theta": 500000.0,
166
+ "sparse_rope_tok_freq_mode": "resample",
167
+ "sparse_rope_tok_theta": 10000.0,
168
+ "sparse_topk_blocks": 16,
169
+ "training_stage": "dash",
170
+ "use_flex_attention": false,
171
+ "use_kl_loss": false,
172
+ "use_sparse_attention": true,
173
+ "use_xformers": false
174
+ },
175
+ "tie_word_embeddings": false,
176
+ "tile_tag": "2D",
177
+ "topk_group": 1,
178
+ "topk_method": "greedy",
179
+ "torch_dtype": "bfloat16",
180
+ "transformers_version": "4.52.1",
181
+ "use_cache": false,
182
+ "use_mla": false,
183
+ "v_head_dim": 32,
184
+ "vision_config": {
185
+ "image_size": 1024,
186
+ "mlp_ratio": 3.7362,
187
+ "model_name": "deeplip_b_l",
188
+ "model_type": "vision",
189
+ "width": {
190
+ "clip-l-14-224": {
191
+ "heads": 4,
192
+ "image_size": 224,
193
+ "layers": 4,
194
+ "patch_size": 14,
195
+ "width": 32
196
+ },
197
+ "sam_vit_b": {
198
+ "downsample_channels": [
199
+ 16,
200
+ 32
201
+ ],
202
+ "global_attn_indexes": [
203
+ 2,
204
+ 5
205
+ ],
206
+ "heads": 4,
207
+ "layers": 6,
208
+ "width": 32,
209
+ "neck_channels": 8
210
+ }
211
+ }
212
+ },
213
+ "vocab_size": 129280,
214
+ "head_dim": 32
215
+ }
configuration_deepseek_v2.py ADDED
@@ -0,0 +1,212 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers.configuration_utils import PretrainedConfig
2
+ from transformers.utils import logging
3
+
4
+ logger = logging.get_logger(__name__)
5
+
6
+ DEEPSEEK_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
7
+ class DeepseekV2Config(PretrainedConfig):
8
+ r"""
9
+ This is the configuration class to store the configuration of a [`DeepseekV2Model`]. It is used to instantiate an DeepSeek
10
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
11
+ defaults will yield a similar configuration to that of the DeepSeek-V2 with multi-latent attention.
12
+
13
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
14
+ documentation from [`PretrainedConfig`] for more information.
15
+
16
+
17
+ Args:
18
+ vocab_size (`int`, *optional*, defaults to 102400):
19
+ Vocabulary size of the Deep model. Defines the number of different tokens that can be represented by the
20
+ `inputs_ids` passed when calling [`DeepseekV2Model`]
21
+ hidden_size (`int`, *optional*, defaults to 4096):
22
+ Dimension of the hidden representations.
23
+ intermediate_size (`int`, *optional*, defaults to 11008):
24
+ Dimension of the MLP representations.
25
+ moe_intermediate_size (`int`, *optional*, defaults to 1407):
26
+ Dimension of the MoE representations.
27
+ num_hidden_layers (`int`, *optional*, defaults to 32):
28
+ Number of hidden layers in the Transformer decoder.
29
+ num_attention_heads (`int`, *optional*, defaults to 32):
30
+ Number of attention heads for each attention layer in the Transformer decoder.
31
+ n_shared_experts (`int`, *optional*, defaults to None):
32
+ Number of shared experts, None means dense model.
33
+ n_routed_experts (`int`, *optional*, defaults to None):
34
+ Number of routed experts, None means dense model.
35
+ routed_scaling_factor (`float`, *optional*, defaults to 1.0):
36
+ Scaling factor or routed experts.
37
+ topk_method (`str`, *optional*, defaults to `gready`):
38
+ Topk method used in routed gate.
39
+ n_group (`int`, *optional*, defaults to None):
40
+ Number of groups for routed experts.
41
+ topk_group (`int`, *optional*, defaults to None):
42
+ Number of selected groups for each token(for each token, ensuring the selected experts is only within `topk_group` groups).
43
+ num_experts_per_tok (`int`, *optional*, defaults to None):
44
+ Number of selected experts, None means dense model.
45
+ moe_layer_freq (`int`, *optional*, defaults to 1):
46
+ The frequency of the MoE layer: one expert layer for every `moe_layer_freq - 1` dense layers.
47
+ first_k_dense_replace (`int`, *optional*, defaults to 0):
48
+ Number of dense layers in shallow layers(embed->dense->dense->...->dense->moe->moe...->lm_head).
49
+ \--k dense layers--/
50
+ norm_topk_prob (`bool`, *optional*, defaults to False):
51
+ Whether to normalize the weights of the routed experts.
52
+ scoring_func (`str`, *optional*, defaults to 'softmax'):
53
+ Method of computing expert weights.
54
+ aux_loss_alpha (`float`, *optional*, defaults to 0.001):
55
+ Auxiliary loss weight coefficient.
56
+ seq_aux = (`bool`, *optional*, defaults to True):
57
+ Whether to compute the auxiliary loss for each individual sample.
58
+ num_key_value_heads (`int`, *optional*):
59
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
60
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
61
+ `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
62
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
63
+ by meanpooling all the original heads within that group. For more details checkout [this
64
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
65
+ `num_attention_heads`.
66
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
67
+ The non-linear activation function (function or string) in the decoder.
68
+ max_position_embeddings (`int`, *optional*, defaults to 2048):
69
+ The maximum sequence length that this model might ever be used with.
70
+ initializer_range (`float`, *optional*, defaults to 0.02):
71
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
72
+ rms_norm_eps (`float`, *optional*, defaults to 1e-06):
73
+ The epsilon used by the rms normalization layers.
74
+ use_cache (`bool`, *optional*, defaults to `True`):
75
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
76
+ relevant if `config.is_decoder=True`.
77
+ pad_token_id (`int`, *optional*):
78
+ Padding token id.
79
+ bos_token_id (`int`, *optional*, defaults to 1):
80
+ Beginning of stream token id.
81
+ eos_token_id (`int`, *optional*, defaults to 2):
82
+ End of stream token id.
83
+ pretraining_tp (`int`, *optional*, defaults to 1):
84
+ Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
85
+ document](https://huggingface.co/docs/transformers/parallelism) to understand more about it. This value is
86
+ necessary to ensure exact reproducibility of the pretraining results. Please refer to [this
87
+ issue](https://github.com/pytorch/pytorch/issues/76232).
88
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
89
+ Whether to tie weight embeddings
90
+ rope_theta (`float`, *optional*, defaults to 10000.0):
91
+ The base period of the RoPE embeddings.
92
+ rope_scaling (`Dict`, *optional*):
93
+ Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
94
+ strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
95
+ `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
96
+ `max_position_embeddings` to the expected new maximum.
97
+ attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
98
+ Whether to use a bias in the query, key, value and output projection layers during self-attention.
99
+ attention_dropout (`float`, *optional*, defaults to 0.0):
100
+ The dropout ratio for the attention probabilities.
101
+ use_mla (`bool`, *optional*, defaults to `True`): Use multi-latent attention or multi-head attention. If True,
102
+ the model will use multi-latent attention, otherwise, it will use multi-head attention.
103
+
104
+ ```python
105
+ >>> from transformers import DeepseekV2Model, DeepseekV2Config
106
+
107
+ >>> # Initializing a Deepseek-V2 style configuration
108
+ >>> configuration = DeepseekV2Config()
109
+
110
+ >>> # Accessing the model configuration
111
+ >>> configuration = model.config
112
+ ```"""
113
+
114
+ model_type = "deepseek_v2"
115
+ keys_to_ignore_at_inference = ["past_key_values"]
116
+
117
+ def __init__(
118
+ self,
119
+ vocab_size=102400,
120
+ hidden_size=4096,
121
+ intermediate_size=11008,
122
+ moe_intermediate_size = 1407,
123
+ num_hidden_layers=30,
124
+ num_attention_heads=32,
125
+ num_key_value_heads=32,
126
+ n_shared_experts = None,
127
+ n_routed_experts = None,
128
+ ep_size = 1,
129
+ routed_scaling_factor = 1.0,
130
+ kv_lora_rank = 512,
131
+ q_lora_rank = 1536,
132
+ qk_rope_head_dim = 64,
133
+ v_head_dim = 128,
134
+ qk_nope_head_dim = 128,
135
+ topk_method = 'gready',
136
+ n_group = None,
137
+ topk_group = None,
138
+ num_experts_per_tok = None,
139
+ moe_layer_freq = 1,
140
+ first_k_dense_replace = 0,
141
+ norm_topk_prob = False,
142
+ scoring_func = 'softmax',
143
+ aux_loss_alpha = 0.001,
144
+ seq_aux = True,
145
+ hidden_act="silu",
146
+ max_position_embeddings=2048,
147
+ initializer_range=0.02,
148
+ rms_norm_eps=1e-6,
149
+ use_cache=True,
150
+ pad_token_id=None,
151
+ bos_token_id=100000,
152
+ eos_token_id=100001,
153
+ pretraining_tp=1,
154
+ tie_word_embeddings=False,
155
+ rope_theta=10000.0,
156
+ rope_scaling=None,
157
+ attention_bias=False,
158
+ attention_dropout=0.0,
159
+ use_mla=True,
160
+ sliding_window=None,
161
+ **kwargs,
162
+ ):
163
+ self.vocab_size = vocab_size
164
+ self.max_position_embeddings = max_position_embeddings
165
+ self.hidden_size = hidden_size
166
+ self.intermediate_size = intermediate_size
167
+ self.moe_intermediate_size = moe_intermediate_size
168
+ self.num_hidden_layers = num_hidden_layers
169
+ self.num_attention_heads = num_attention_heads
170
+ self.n_shared_experts = n_shared_experts
171
+ self.n_routed_experts = n_routed_experts
172
+ self.ep_size = ep_size
173
+ self.routed_scaling_factor = routed_scaling_factor
174
+ self.kv_lora_rank = kv_lora_rank
175
+ self.q_lora_rank = q_lora_rank
176
+ self.qk_rope_head_dim = qk_rope_head_dim
177
+ self.v_head_dim = v_head_dim
178
+ self.qk_nope_head_dim = qk_nope_head_dim
179
+ self.topk_method = topk_method
180
+ self.n_group = n_group
181
+ self.topk_group = topk_group
182
+ self.num_experts_per_tok = num_experts_per_tok
183
+ self.moe_layer_freq = moe_layer_freq
184
+ self.first_k_dense_replace = first_k_dense_replace
185
+ self.norm_topk_prob = norm_topk_prob
186
+ self.scoring_func = scoring_func
187
+ self.aux_loss_alpha = aux_loss_alpha
188
+ self.seq_aux = seq_aux
189
+ # for backward compatibility
190
+ if num_key_value_heads is None:
191
+ num_key_value_heads = num_attention_heads
192
+
193
+ self.num_key_value_heads = num_key_value_heads
194
+ self.hidden_act = hidden_act
195
+ self.initializer_range = initializer_range
196
+ self.rms_norm_eps = float(rms_norm_eps)
197
+ self.pretraining_tp = pretraining_tp
198
+ self.use_cache = use_cache
199
+ self.rope_theta = rope_theta
200
+ self.rope_scaling = rope_scaling
201
+ self.attention_bias = attention_bias
202
+ self.attention_dropout = attention_dropout
203
+ self.use_mla = use_mla
204
+ self.sliding_window = sliding_window
205
+
206
+ super().__init__(
207
+ pad_token_id=pad_token_id,
208
+ bos_token_id=bos_token_id,
209
+ eos_token_id=eos_token_id,
210
+ tie_word_embeddings=tie_word_embeddings,
211
+ **kwargs,
212
+ )
conversation.py ADDED
@@ -0,0 +1,280 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ From https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py
3
+ """
4
+
5
+ import dataclasses
6
+ from enum import IntEnum, auto
7
+ from typing import Any, Dict, List
8
+
9
+
10
+ class SeparatorStyle(IntEnum):
11
+ """Separator styles."""
12
+
13
+ DeepSeek = auto()
14
+ DeepSeekV2 = auto()
15
+ PLAIN = auto()
16
+ ALIGNMENT = auto()
17
+
18
+
19
+ @dataclasses.dataclass
20
+ class Conversation:
21
+ """A class that manages prompt templates and keeps all conversation history."""
22
+
23
+ # The name of this template
24
+ name: str
25
+ # The template of the system prompt
26
+ system_template: str = "{system_message}"
27
+ # The system message
28
+ system_message: str = ""
29
+ # The names of two roles
30
+ roles: List[str] = (("USER", "ASSISTANT"),)
31
+ # All messages. Each item is (role, message).
32
+ messages: List[List[str]] = ()
33
+ # The number of few shot examples
34
+ offset: int = 0
35
+ # The separator style and configurations
36
+ sep_style: SeparatorStyle = SeparatorStyle.DeepSeek
37
+ sep: str = "\n"
38
+ sep2: str = None
39
+ # Stop criteria (the default one is EOS token)
40
+ stop_str: str = None
41
+ # Stops generation if meeting any token in this list
42
+ stop_token_ids: List[int] = None
43
+
44
+ def get_prompt(self) -> str:
45
+ """Get the prompt for generation."""
46
+ system_prompt = self.system_template.format(system_message=self.system_message)
47
+ if self.sep_style == SeparatorStyle.DeepSeek:
48
+ seps = [self.sep, self.sep2]
49
+ if system_prompt == "" or system_prompt is None:
50
+ ret = ""
51
+ else:
52
+ ret = system_prompt + seps[0]
53
+ for i, (role, message) in enumerate(self.messages):
54
+ if message:
55
+ ret += role + ": " + message + seps[i % 2]
56
+ else:
57
+ ret += role + ":"
58
+ return ret
59
+ elif self.sep_style == SeparatorStyle.DeepSeekV2:
60
+ seps = [self.sep, self.sep2]
61
+ if system_prompt == "" or system_prompt is None:
62
+ ret = ""
63
+ else:
64
+ ret = system_prompt + seps[0]
65
+ for i, (role, message) in enumerate(self.messages):
66
+ if message:
67
+ if role == "User":
68
+ ret += "<|sft▁begin|>\n" + message + self.sep #<|sft▁begin|>User Input<|sft▁end|>\nResponse<|end▁of▁sentence|>
69
+ else:
70
+ ret += message + self.sep2
71
+ else:
72
+ ret = ret
73
+ return ret
74
+
75
+ elif self.sep_style == SeparatorStyle.PLAIN:
76
+ seps = [self.sep, self.sep2]
77
+ ret = ""
78
+ for i, (role, message) in enumerate(self.messages):
79
+ if message:
80
+ if type(message) is tuple:
81
+ message, _, _ = message
82
+ if i % 2 == 0:
83
+ ret += message + seps[i % 2]
84
+ else:
85
+ ret += message + seps[i % 2]
86
+ else:
87
+ ret += ""
88
+ return ret
89
+ elif self.sep_style == SeparatorStyle.ALIGNMENT:
90
+ seps = [self.sep, self.sep2]
91
+ ret = ""
92
+ for i, (role, message) in enumerate(self.messages):
93
+ if message:
94
+ if type(message) is tuple:
95
+ message, _, _ = message
96
+ if i % 2 == 0:
97
+ ret += '<image>\n' + seps[i % 2]
98
+ else:
99
+ ret += message + seps[i % 2]
100
+ else:
101
+ ret += ""
102
+ return ret
103
+ else:
104
+ raise ValueError(f"Invalid style: {self.sep_style}")
105
+
106
+ def set_system_message(self, system_message: str):
107
+ """Set the system message."""
108
+ self.system_message = system_message
109
+
110
+ def append_message(self, role: str, message: str):
111
+ """Append a new message."""
112
+ self.messages.append([role, message])
113
+
114
+ def update_last_message(self, message: str):
115
+ """Update the last output.
116
+
117
+ The last message is typically set to be None when constructing the prompt,
118
+ so we need to update it in-place after getting the response from a model.
119
+ """
120
+ self.messages[-1][1] = message
121
+
122
+ def reset_message(self):
123
+ """Reset a new message."""
124
+ self.messages = []
125
+
126
+ def to_gradio_chatbot(self):
127
+ """Convert the conversation to gradio chatbot format."""
128
+ ret = []
129
+ for i, (role, msg) in enumerate(self.messages[self.offset :]):
130
+ if i % 2 == 0:
131
+ ret.append([msg, None])
132
+ else:
133
+ ret[-1][-1] = msg
134
+ return ret
135
+
136
+ def to_openai_api_messages(self):
137
+ """Convert the conversation to OpenAI chat completion format."""
138
+ system_prompt = self.system_template.format(system_message=self.system_message)
139
+ ret = [{"role": "system", "content": system_prompt}]
140
+
141
+ for i, (_, msg) in enumerate(self.messages[self.offset :]):
142
+ if i % 2 == 0:
143
+ ret.append({"role": "user", "content": msg})
144
+ else:
145
+ if msg is not None:
146
+ ret.append({"role": "assistant", "content": msg})
147
+ return ret
148
+
149
+ def copy(self):
150
+ return Conversation(
151
+ name=self.name,
152
+ system_template=self.system_template,
153
+ system_message=self.system_message,
154
+ roles=self.roles,
155
+ messages=[[x, y] for x, y in self.messages],
156
+ offset=self.offset,
157
+ sep_style=self.sep_style,
158
+ sep=self.sep,
159
+ sep2=self.sep2,
160
+ stop_str=self.stop_str,
161
+ stop_token_ids=self.stop_token_ids,
162
+ )
163
+
164
+ def dict(self):
165
+ return {
166
+ "template_name": self.name,
167
+ "system_message": self.system_message,
168
+ "roles": self.roles,
169
+ "messages": self.messages,
170
+ "offset": self.offset,
171
+ }
172
+
173
+
174
+ # A global registry for all conversation templates
175
+ conv_templates: Dict[str, Conversation] = {}
176
+
177
+
178
+ def register_conv_template(template: Conversation, override: bool = False):
179
+ """Register a new conversation template."""
180
+ if not override:
181
+ assert template.name not in conv_templates, f"{template.name} has been registered."
182
+
183
+ conv_templates[template.name] = template
184
+
185
+
186
+ def get_conv_template(name: str) -> Conversation:
187
+ """Get a conversation template."""
188
+ return conv_templates[name].copy()
189
+
190
+
191
+ register_conv_template(
192
+ Conversation(
193
+ name="deepseek",
194
+ system_template="{system_message}",
195
+ # system_message="You are a helpful assistant. Please answer truthfully and write out your "
196
+ # "thinking step by step to be sure you get the right answer.",
197
+ system_message="",
198
+ roles=("<|User|>", "<|Assistant|>"),
199
+ messages=(),
200
+ offset=0,
201
+ sep_style=SeparatorStyle.DeepSeek,
202
+ sep="\n\n",
203
+ sep2="<|end▁of▁sentence|>",
204
+ stop_token_ids=[100001],
205
+ stop_str=["User:", "<|end▁of▁sentence|>"]
206
+ )
207
+ )
208
+ register_conv_template(
209
+ Conversation(
210
+ name="deepseekv2",
211
+ system_template="{system_message}",
212
+ # system_message="You are a helpful assistant. Please answer truthfully and write out your "
213
+ # "thinking step by step to be sure you get the right answer.",
214
+ system_message="",
215
+ roles=("<|User|>", "<|Assistant|>"),
216
+ messages=(),
217
+ offset=0,
218
+ sep_style=SeparatorStyle.DeepSeek,
219
+ sep="",
220
+ sep2="<|end▁of▁sentence|>",
221
+ stop_token_ids=[100001],
222
+ stop_str=["User:", "<|end▁of▁sentence|>"]
223
+ )
224
+ )
225
+
226
+
227
+ register_conv_template(
228
+ Conversation(
229
+ name="plain",
230
+ system_template="",
231
+ system_message="",
232
+ roles=("", ""),
233
+ messages=(),
234
+ offset=0,
235
+ sep_style=SeparatorStyle.PLAIN,
236
+ sep="",
237
+ sep2="",
238
+ stop_token_ids=[100001],
239
+ stop_str=['</s>'],
240
+ )
241
+ )
242
+
243
+
244
+ register_conv_template(
245
+ Conversation(
246
+ name="alignment",
247
+ system_template="",
248
+ system_message="",
249
+ roles=("", ""),
250
+ messages=(),
251
+ offset=0,
252
+ sep_style=SeparatorStyle.ALIGNMENT,
253
+ sep="",
254
+ sep2="",
255
+ stop_token_ids=[100001],
256
+ stop_str=['</s>'],
257
+ )
258
+ )
259
+
260
+
261
+ if __name__ == "__main__":
262
+ print("deepseek template:")
263
+ conv = get_conv_template("deepseek")
264
+ conv.append_message(conv.roles[0], "Hello!")
265
+ conv.append_message(conv.roles[1], "Hi! This is Tony.")
266
+ conv.append_message(conv.roles[0], "Who are you?")
267
+ conv.append_message(conv.roles[1], "I am a helpful assistant.")
268
+ conv.append_message(conv.roles[0], "How are you?")
269
+ conv.append_message(conv.roles[1], None)
270
+ print(conv.get_prompt())
271
+
272
+ print("deepseekv2 template:")
273
+ conv = get_conv_template("deepseekv2")
274
+ conv.append_message(conv.roles[0], "Hello!")
275
+ conv.append_message(conv.roles[1], "Hi! This is Tony.")
276
+ conv.append_message(conv.roles[0], "Who are you?")
277
+ conv.append_message(conv.roles[1], "I am a helpful assistant.")
278
+ conv.append_message(conv.roles[0], "How are you?")
279
+ conv.append_message(conv.roles[1], None)
280
+ print(conv.get_prompt())
dash_attention.py ADDED
@@ -0,0 +1,1349 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Pure-PyTorch, fully-differentiable DASH attention for Unlimited-OCR.
2
+
3
+ VARIANT B: entmax routing + HARD top-k budget + PER-LAYER entmax temperature
4
+ (see the banner below the imports for the full B-vs-base rationale).
5
+
6
+ DASH = Differentiable and Adaptive Sparse Hierarchical attention
7
+ (https://arxiv.org/abs/2605.18753). Unlike the Lightning-Indexer + top-k DSA
8
+ path of the base implementation (whose block selection is DISCRETE and therefore
9
+ needs a separate warmup + KL-distillation stage to train the indexer), DASH
10
+ routes with **1.5-entmax**, which yields a sparse *and differentiable*
11
+ distribution over key blocks. Gradient reaches the router directly from the LM
12
+ loss, so no KL teacher / no two-stage curriculum is required.
13
+
14
+ This module deliberately uses **plain PyTorch ops only** (no Triton). The only
15
+ custom autograd is the tiny 1.5-entmax Jacobian (a standard, well-known
16
+ closed form over the block axis) -- everything else is ordinary autograd.
17
+ This is the "correctness-first" realization: it is O(S^2) in the attention
18
+ stage (computed in query chunks to bound peak memory), so it is meant for
19
+ training / convergence validation, not for maximum throughput. A fused sparse
20
+ kernel is a later optimization once the recipe is validated.
21
+
22
+ Three stages (mirroring the paper and ``dash-attention-main``):
23
+ stage1 summarize_blocks : one learned key summary per KV block, via a
24
+ softmax attention with a learnable ``head_cls``.
25
+ stage2 route : query->summary scores -> 1.5-entmax over blocks
26
+ (adaptive, differentiable sparse "prior").
27
+ stage3 prior_biased_attn: token-level softmax attention restricted to the
28
+ routed blocks UNION the local window, with
29
+ ``log(prior)`` injected as an additive bias so the
30
+ hierarchy stays differentiable end-to-end.
31
+
32
+ Optional image-block restriction: when an image-block mask is supplied, the routing
33
+ candidate set is restricted to image blocks (any page), matching the
34
+ ``image_bucket_constraint=False`` semantics of the base implementation.
35
+ """
36
+
37
+ import math
38
+ import os
39
+
40
+ import torch
41
+ import torch.nn as nn
42
+ import torch.nn.functional as F
43
+
44
+ _DASH_DEBUG = os.environ.get("DASH_DEBUG_ANOMALY") == "1"
45
+
46
+ # ===========================================================================
47
+ # VARIANT B -- entmax + hard top-k + per-layer temperature
48
+ # (overlay on patch_config/sparse_dash_code)
49
+ #
50
+ # Differences vs the base DASH overlay:
51
+ # * The attended set is a HARD top-k, k annealed 64 -> 12 (`_KSchedule`), so
52
+ # every query in every layer reads exactly k remote blocks. The base overlay
53
+ # attended the entmax support, whose size varied per layer (13-27 at a mean
54
+ # of 16) and per step.
55
+ # * entmax is KEPT: it still produces the differentiable prior whose
56
+ # mean-centered log is the routing bias, so this stays faithful to the paper's
57
+ # adaptive weighting -- only the cardinality is now fixed.
58
+ # * Each layer runs its OWN closed loop on its OWN entmax temperature, with the
59
+ # current k as setpoint (`_PerLayerScheduler`). Once the count is fixed the
60
+ # temperature's job changes: it has to keep each layer's entmax support ~= k,
61
+ # otherwise the kept blocks either sit at the prior floor (support << k, no
62
+ # gradient) or get an almost flat prior (support >> k, no ranking signal).
63
+ # The base overlay's single global scale could only pin the across-layer mean.
64
+ # * Per-layer one-shot calibration replaces the leader-only calibration, so
65
+ # every layer starts at the initial k instead of inheriting layer 0's
66
+ # temperature.
67
+ # Variant A (patch_config/dash_softmax_topk) is the same budget without entmax.
68
+ # ===========================================================================
69
+ # DASH_LOG_SPARSITY=1 -> print entmax routing sparsity (how many blocks/tokens
70
+ # the router keeps vs drops) during training, throttled to every
71
+ # DASH_LOG_SPARSITY_EVERY calls OF EACH LAYER (so every layer shows up in the log;
72
+ # a single global counter aliased against the layer count and only ever printed
73
+ # the same few layers).
74
+ _LOG_SPARSITY = os.environ.get("DASH_LOG_SPARSITY", "0") == "1"
75
+ _SPARSITY_EVERY = max(1, int(os.environ.get("DASH_LOG_SPARSITY_EVERY", "50")))
76
+ _sparsity_calls = {}
77
+
78
+
79
+ def _log_train_sparsity(prior, cand, valid, block_size, sid=0):
80
+ """Print this layer's entmax support vs the hard k budget it is aimed at.
81
+
82
+ ``attn`` is what the layer actually reads (== min(k, #candidates), guaranteed
83
+ by the top-k); ``act`` is the entmax support size, the quantity this layer's
84
+ own closed loop drives onto k. act << attn means most kept blocks sit at the
85
+ prior floor and carry no gradient; act >> attn means the prior is flat over
86
+ the kept set. Both are per-layer, hence the L<sid> tag.
87
+ """
88
+ n = _sparsity_calls.get(sid, 0) + 1
89
+ _sparsity_calls[sid] = n
90
+ # Always print each layer's FIRST call: with EVERY=N the first line of a layer
91
+ # would otherwise take N micro-batches, so a run that dies early (or a short
92
+ # smoke test) showed no per-layer sparsity at all.
93
+ if n != 1 and n % _SPARSITY_EVERY != 0:
94
+ return
95
+ with torch.no_grad():
96
+ nblk = prior.shape[-1]
97
+ c = cand.sum(dim=-1).float() # causal candidate blocks/query
98
+ a = ((prior > 0) & cand).sum(dim=-1).float() # entmax support/query
99
+ t = valid.sum(dim=-1).float() # blocks actually attended
100
+ cm, am, tm = c.mean().item(), a.mean().item(), t.mean().item()
101
+ drop = (1.0 - t.sum() / c.clamp(min=1).sum()).item() * 100.0
102
+ print(f"[DASH sparsity L{sid}] call#{n}: attn {tm:.1f}/{cm:.1f} cand "
103
+ f"blocks/query (k={_KS.k}, drop {drop:.1f}%); entmax act={am:.1f}; "
104
+ f"total_blocks={nblk}; attn ~{tm * block_size:.0f} remote tok/query "
105
+ f"(block_size={block_size}){_SCHED.state_str(sid)}", flush=True)
106
+
107
+
108
+ # DASH_LOG_NEFF_EVERY>0 -> print, per layer, how many of the k gathered blocks
109
+ # the ATTENTION actually uses. `sparsity` only reports how many blocks the
110
+ # ROUTER selected; it cannot see that the log(prior) bias spans ~9 (floor 1e-4,
111
+ # sigma 1) while qk/sqrt(d) spans ~3-6, so the bias is free to veto a block
112
+ # top-k already paid to read. n_eff = exp(H(w)) with w = each block's share of
113
+ # the remote attention mass: n_eff ~= 2 at k=12 means 10 of the 12 gathered
114
+ # blocks are dead weight -- a failure the temperature loop cannot observe.
115
+ # remote_mass = the remote blocks' share of (remote + local window) mass; n_eff
116
+ # means nothing when the row spends all its mass on the free local context.
117
+ # Cost: one no_grad [R, K*block_size] qk per print. ROWS query rows are sampled
118
+ # and processed CHUNK rows at a time, because these temporaries live alongside
119
+ # the training activations (S=32k is already tight).
120
+ _LOG_NEFF_EVERY = max(0, int(os.environ.get("DASH_LOG_NEFF_EVERY", "0")))
121
+ _LOG_NEFF_ROWS = max(1, int(os.environ.get("DASH_LOG_NEFF_ROWS", "64")))
122
+ _LOG_NEFF_CHUNK = max(1, int(os.environ.get("DASH_LOG_NEFF_CHUNK", "8")))
123
+ _neff_calls = {}
124
+
125
+
126
+ def _log_neff(q, k, topi, blk_bias, valid, position_ids, block_size,
127
+ local_window, scale, sid=0):
128
+ """Effective block utilisation of the k blocks this layer gathered.
129
+
130
+ Same throttling convention as _log_train_sparsity: per-layer counter, the
131
+ first call of every layer always prints. Batch element 0 only.
132
+ """
133
+ n = _neff_calls.get(sid, 0) + 1
134
+ _neff_calls[sid] = n
135
+ if n != 1 and n % _LOG_NEFF_EVERY != 0:
136
+ return
137
+ with torch.no_grad():
138
+ _, H, S, D = q.shape
139
+ K = topi.shape[-1]
140
+ b = 0
141
+ rows = torch.nonzero(valid[b].any(dim=-1)).flatten()
142
+ if rows.numel() == 0:
143
+ print(f"[DASH n_eff L{sid}] call#{n}: no remote selection "
144
+ f"(k={_KS.k})", flush=True)
145
+ return
146
+ if rows.numel() > _LOG_NEFF_ROWS:
147
+ rows = rows[::max(1, rows.numel() // _LOG_NEFF_ROWS)][:_LOG_NEFF_ROWS]
148
+ ar = torch.arange(block_size, device=q.device)
149
+ aw = torch.arange(local_window, device=q.device)
150
+ neff = top1 = rmass = 0.0
151
+ for c0 in range(0, rows.numel(), _LOG_NEFF_CHUNK):
152
+ r = rows[c0:c0 + _LOG_NEFF_CHUNK]
153
+ R = r.numel()
154
+ qpos = position_ids[b, r].long() # [R]
155
+ qi = q[b][:, r, :] # [H,R,D]
156
+ # remote: the exact blocks the kernel/dense path reads, same bias.
157
+ pos = (topi[b][r].clamp_min(0)[:, :, None] * block_size + ar
158
+ ).reshape(R, -1) # [R,K*bs]
159
+ keep = ((pos <= qpos[:, None]) & (pos < S)
160
+ & valid[b][r][:, :, None].expand(R, K, block_size
161
+ ).reshape(R, -1))
162
+ kk = k[b][:, pos.clamp(0, S - 1).reshape(-1), :].view(
163
+ H, R, K * block_size, D)
164
+ lg = torch.einsum("hrd,hrnd->hrn", qi, kk).float() * scale
165
+ lg = lg + blk_bias[b][r].float()[:, :, None].expand(
166
+ R, K, block_size).reshape(R, -1)
167
+ lg = lg.masked_fill(~keep, float("-inf"))
168
+ # local window: free context the remote blocks compete against.
169
+ lpos = qpos[:, None] - aw # [R,W]
170
+ lkk = k[b][:, lpos.clamp_min(0).reshape(-1), :].view(
171
+ H, R, local_window, D)
172
+ llg = torch.einsum("hrd,hrnd->hrn", qi, lkk).float() * scale
173
+ llg = llg.masked_fill(~(lpos >= 0), float("-inf"))
174
+ # shared max: remote and local must be softmaxed in one denominator.
175
+ m = torch.maximum(lg.amax(-1), llg.amax(-1)).clamp_min(-1e30)
176
+ er = torch.nan_to_num(torch.exp(lg - m[..., None]))
177
+ el = torch.nan_to_num(torch.exp(llg - m[..., None]))
178
+ zr = er.view(H, R, K, block_size).sum(-1) # [H,R,K]
179
+ wr = zr / zr.sum(-1, keepdim=True).clamp_min(1e-20)
180
+ ent = -(wr * torch.log(wr.clamp_min(1e-20))).sum(-1) # [H,R]
181
+ neff += torch.exp(ent).sum().item()
182
+ top1 += wr.amax(-1).sum().item()
183
+ tot = (er.sum(-1) + el.sum(-1)).clamp_min(1e-20)
184
+ rmass += (er.sum(-1) / tot).sum().item()
185
+ cnt = float(H * rows.numel())
186
+ print(f"[DASH n_eff L{sid}] call#{n}: n_eff={neff / cnt:.2f} of K={K} "
187
+ f"(k={_KS.k}); top1_blk={top1 / cnt:.3f}; "
188
+ f"remote_mass={rmass / cnt:.3f}; rows={rows.numel()}", flush=True)
189
+
190
+
191
+ # DASH_LOG_RECALL_EVERY>0 -> per layer, the paper-style ORACLE recall of the
192
+ # router's top-k selection (same definition as the DSA report's Fig.3 and the
193
+ # `_log_indexer_recall` of the base implementation):
194
+ #
195
+ # I* = oracle top-k blocks, ranked by the DENSE attention mass P_b that the
196
+ # main branch (q@k) puts on block b.
197
+ # I_hat = the blocks the router actually selected (`topi`, top-k over the prior).
198
+ # block recall@k = |I* ^ I_hat| / k
199
+ # score recall@k = sum_{b in I* ^ I_hat} P_b / sum_{b in I*} P_b
200
+ #
201
+ # Both are measured on the SAME domain the selector uses (candidate blocks only:
202
+ # causal, outside the local window, image-constrained if dash_image_only), so
203
+ # P_b is a softmax normalized over the candidate tokens and mass_all == 1.
204
+ # student_kept / teacher_kept are the same numbers without the intersection:
205
+ # what fraction of the candidate mass the router's k blocks capture vs. what the
206
+ # best possible k blocks capture -- their gap is the pure ranking loss, which is
207
+ # easier to read than either absolute value.
208
+ # Only CONTESTED rows count (n_cand > k); when cand <= k the top-k trivially
209
+ # takes everything and recall is 1.0 by construction, which would inflate the
210
+ # mean. Chance level is ~k/cand_q, so always read recall against the printed
211
+ # cand/q.
212
+ # Cost: one no_grad [H, chunk, S] qk per print (the full dense row, unlike n_eff
213
+ # which only re-reads the k gathered blocks), so keep EVERY large and ROWS small.
214
+ _LOG_RECALL_EVERY = max(0, int(os.environ.get("DASH_LOG_RECALL_EVERY", "0")))
215
+ _LOG_RECALL_ROWS = max(1, int(os.environ.get("DASH_LOG_RECALL_ROWS", "32")))
216
+ _LOG_RECALL_CHUNK = max(1, int(os.environ.get("DASH_LOG_RECALL_CHUNK", "4")))
217
+ _recall_calls = {}
218
+
219
+
220
+ def _log_oracle_recall(q, k, topi, valid, cand, position_ids, block_size,
221
+ scale, sid=0):
222
+ """Oracle (dense-attention) recall of this layer's routed block set.
223
+
224
+ Same throttling convention as the other diagnostics (per-layer counter, each
225
+ layer's first call always prints). Batch element 0 only, rank 0 only unless
226
+ DASH_LOG_RECALL_ALL_RANKS=1.
227
+ """
228
+ n = _recall_calls.get(sid, 0) + 1
229
+ _recall_calls[sid] = n
230
+ if n != 1 and n % _LOG_RECALL_EVERY != 0:
231
+ return
232
+ if os.environ.get("DASH_LOG_RECALL_ALL_RANKS", "0") != "1":
233
+ import torch.distributed as dist
234
+ if dist.is_available() and dist.is_initialized() and dist.get_rank() != 0:
235
+ return
236
+ with torch.no_grad():
237
+ _, H, S, D = q.shape
238
+ K = topi.shape[-1]
239
+ b = 0
240
+ nblk = cand.shape[-1]
241
+ pad = nblk * block_size - S
242
+ # A row is only meaningful if the router had a real choice to make.
243
+ ncand = cand[b].sum(dim=-1) # [S]
244
+ rows = torch.nonzero((ncand > K) & valid[b].any(dim=-1)).flatten()
245
+ if rows.numel() == 0:
246
+ print(f"[DASH recall L{sid}] call#{n}: no contested row "
247
+ f"(k={_KS.k}, K={K})", flush=True)
248
+ return
249
+ if rows.numel() > _LOG_RECALL_ROWS:
250
+ rows = rows[::max(1, rows.numel() // _LOG_RECALL_ROWS)][:_LOG_RECALL_ROWS]
251
+ kpos = torch.arange(S, device=q.device)
252
+ br = sr = kept = teach = t1 = 0.0
253
+ cnt = 0
254
+ for c0 in range(0, rows.numel(), _LOG_RECALL_CHUNK):
255
+ r = rows[c0:c0 + _LOG_RECALL_CHUNK]
256
+ R = r.numel()
257
+ qpos = position_ids[b, r].long() # [R]
258
+ cnd = cand[b][r] # [R,nblk]
259
+ # candidate blocks -> candidate tokens, then re-impose causality
260
+ # (the last candidate block can extend past the query for pad rows).
261
+ keep = cnd.repeat_interleave(block_size, dim=-1)[:, :S] \
262
+ & (kpos[None, :] <= qpos[:, None]) # [R,S]
263
+ lg = torch.einsum("hrd,hsd->hrs", q[b][:, r, :], k[b]).float() * scale
264
+ lg = lg.masked_fill(~keep[None], float("-inf"))
265
+ p = torch.softmax(lg, dim=-1)
266
+ p = torch.nan_to_num(p) # empty rows
267
+ if pad:
268
+ p = F.pad(p, (0, pad), value=0.0)
269
+ # router selection is head-shared, so the oracle is the head-mean
270
+ # block distribution (one teacher per query row).
271
+ p_blk = p.view(H, R, nblk, block_size).sum(-1).mean(0) # [R,nblk]
272
+ p_msk = p_blk.masked_fill(~cnd, -1.0)
273
+ star = p_msk.topk(K, dim=-1).indices
274
+ star_m = torch.zeros_like(cnd).scatter_(-1, star, True) & cnd
275
+ # invalid (padding) selection slots are routed to a scratch column so
276
+ # they cannot be mistaken for a hit on block 0.
277
+ hidx = torch.where(valid[b][r], topi[b][r].clamp_min(0),
278
+ torch.full_like(topi[b][r], nblk))
279
+ hat_m = torch.zeros((R, nblk + 1), dtype=torch.bool, device=q.device
280
+ ).scatter_(-1, hidx, True)[:, :nblk] & cnd
281
+ inter = star_m & hat_m
282
+ m_all = (p_blk * cnd).sum(-1).clamp_min(1e-9)
283
+ m_star = (p_blk * star_m).sum(-1)
284
+ ok = m_star > 0
285
+ br += (inter.sum(-1).float() / K)[ok].sum().item()
286
+ sr += ((p_blk * inter).sum(-1) / m_star.clamp_min(1e-9))[ok].sum().item()
287
+ kept += ((p_blk * hat_m).sum(-1) / m_all)[ok].sum().item()
288
+ teach += (m_star / m_all)[ok].sum().item()
289
+ t1 += (p_msk.argmax(-1) == topi[b][r][:, 0])[ok].float().sum().item()
290
+ cnt += int(ok.sum().item())
291
+ if cnt == 0:
292
+ return
293
+ print(f"[DASH recall L{sid}] call#{n}: k={_KS.k} K={K} "
294
+ f"block_recall@k={br / cnt:.4f} score_recall@k={sr / cnt:.4f}; "
295
+ f"student_kept={kept / cnt:.4f} oracle_kept={teach / cnt:.4f} "
296
+ f"top1={t1 / cnt:.4f}; cand/q={ncand[rows].float().mean().item():.1f} "
297
+ f"contested_rows={cnt}/{rows.numel()} nblk={nblk}", flush=True)
298
+
299
+
300
+ # ---------------------------------------------------------------------------
301
+ # Sparsity scheduler -- opt-in via DASH_SPARSITY_SCHED=1 (default OFF, in which
302
+ # case every value below is ignored and behaviour is bit-identical to before).
303
+ #
304
+ # WHY the entmax scale is the sparsity knob: `_route` feeds
305
+ # block_scores * dash_scaling_factor / sqrt(index_dim)
306
+ # to 1.5-entmax, and entmax's support size is governed by the ABSOLUTE scale of
307
+ # its input (scale -> 0 degenerates to uniform/dense, scale up shrinks the
308
+ # support). The shipped 1.0/sqrt(128) ~= 0.088 flattens the score gaps, so the
309
+ # prior stays near-dense. Ramping the scale up = annealing a temperature down.
310
+ #
311
+ # WHY prior_floor must follow: the router's only gradient channel is
312
+ # bias = (log(clamp(p, prior_floor)) - mu) / sigma
313
+ # If the active p_j fall below `prior_floor`, every logp clamps to the same
314
+ # log(floor), the mean-centered bias collapses to 0 and the router stops
315
+ # learning. So the floor is annealed DOWN geometrically as the prior sharpens.
316
+ #
317
+ # Step unit = one training micro-batch forward. Gradient-checkpoint recompute
318
+ # passes are detected (they run inside a backward graph task) and NOT counted,
319
+ # and only the first-registered layer advances the clock, so every DASH layer
320
+ # shares one schedule.
321
+ # ---------------------------------------------------------------------------
322
+ # ---------------------------------------------------------------------------
323
+ # ---------------------------------------------------------------------------
324
+ _LOG_PAGE_EVERY = max(0, int(os.environ.get("DASH_LOG_PAGE_EVERY", "0")))
325
+ _LOG_PAGE_ROWS = max(1, int(os.environ.get("DASH_LOG_PAGE_ROWS", "64")))
326
+
327
+
328
+ @torch.no_grad()
329
+ def _log_page_purity(topi, valid, cand, remote, img_page_ids, query_pages,
330
+ block_size, nblk, sid=0, n_call=0):
331
+ if img_page_ids is None or query_pages is None:
332
+ return
333
+ S = topi.shape[1]
334
+ step = max(1, S // _LOG_PAGE_ROWS)
335
+ rows = torch.arange(0, S, step, device=topi.device)[:_LOG_PAGE_ROWS]
336
+ ip = img_page_ids[:1]
337
+ pad = nblk * block_size - ip.shape[-1]
338
+ if pad > 0:
339
+ ip = F.pad(ip, (0, pad), value=-1)
340
+ ip_blk = ip[..., :nblk * block_size].view(nblk, block_size)
341
+ ti, vi = topi[0, rows], valid[0, rows] # [R,K]
342
+ qp = query_pages[0, rows][:, None, None] # [R,1,1]
343
+ bp = ip_blk[ti] # [R,K,block_size]
344
+ is_img = bp >= 0
345
+ n = int(vi.sum().clamp(min=1))
346
+ nonimg = int(((is_img.sum(-1) == 0) & vi).sum())
347
+ other = (is_img & (bp != qp)).sum(-1)
348
+ same = ((bp == qp) & (qp >= 0)).sum(-1)
349
+ cross = int(((other > 0) & vi).sum())
350
+ pure = int(((other > 0) & (same == 0) & vi).sum())
351
+ empty = float((~(cand[0, rows] & remote[0, rows]).any(-1)).float().mean() * 100)
352
+ no_page = float((query_pages[0, rows] < 0).float().mean() * 100)
353
+ print(f"[DASH page L{sid}] call#{n_call}: rows={len(rows)} sel={n} "
354
+ f"non_image={100 * nonimg / n:.2f}% cross_block={100 * cross / n:.2f}% "
355
+ f"pure_cross={100 * pure / n:.2f}% cand_empty={empty:.1f}% "
356
+ f"no_page={no_page:.1f}%", flush=True)
357
+
358
+
359
+ _SCHED_ON = os.environ.get("DASH_SPARSITY_SCHED", "0") == "1"
360
+
361
+
362
+ def _envf(name, default):
363
+ v = os.environ.get(name)
364
+ return default if v is None or v == "" else float(v)
365
+
366
+
367
+ def _envi(name, default):
368
+ v = os.environ.get(name)
369
+ return default if v is None or v == "" else int(v)
370
+
371
+
372
+ # ---------------------------------------------------------------------------
373
+ # Hard top-k budget with a dense-to-sparse k ANNEAL.
374
+ #
375
+ # The routed-block count is a HARD, deterministic budget: every query attends
376
+ # min(k, #candidates) remote blocks in EVERY layer, so "same number of active
377
+ # blocks in every layer at every step" holds by construction. entmax still runs
378
+ # (it produces the differentiable weights on the kept blocks), and each layer's
379
+ # entmax temperature is closed-looped onto this same k -- see
380
+ # `_PerLayerScheduler` for why that matters once the count is fixed.
381
+ #
382
+ # k anneals geometrically K_START -> K_END over K_ANNEAL_STEPS optimizer steps.
383
+ # WHY anneal at all when the budget is exact: the base checkpoint was pretrained
384
+ # dense, and 12 blocks is ~2.3% of the KV at 16k. Truncating that hard at step 0
385
+ # with lr=1e-4 full-finetune and no warmup perturbs the representation before the
386
+ # router has learned to rank blocks; starting wide gives the router a usable
387
+ # gradient signal while the model is still close to its pretrained state.
388
+ #
389
+ # k MUST be constant for a whole micro-batch: with gradient checkpointing the
390
+ # layers are recomputed in backward, and a k that moved mid-forward changes the
391
+ # gathered width -> `CheckpointError: recomputed values have different metadata`.
392
+ # So it is refreshed ONLY at a micro-batch edge, by the leader layer.
393
+ # ---------------------------------------------------------------------------
394
+ class _KSchedule:
395
+ # gc scans are not free, so only probe for the TrainerState a few times.
396
+ _PROBE_AT = (1, 4, 16, 64, 256)
397
+
398
+ def __init__(self):
399
+ self.k_start = _envi("DASH_K_START", 64)
400
+ self.k_end = _envi("DASH_K_END", 12)
401
+ self.steps = max(1, _envi("DASH_K_ANNEAL_STEPS", 1000))
402
+ self.log_every = max(1, _envi("DASH_K_LOG_EVERY", 50))
403
+ self._k = self.k_start
404
+ self._micro = 0
405
+ self._step = 0
406
+ self._state = None
407
+ self._leader = None
408
+ self._n_reg = 0
409
+ self._logged = False
410
+
411
+ def register(self):
412
+ """Returns an id; the first caller becomes the clock leader."""
413
+ sid = self._n_reg
414
+ self._n_reg += 1
415
+ if self._leader is None:
416
+ self._leader = sid
417
+ return sid
418
+
419
+ def _probe_state(self):
420
+ """Locate the live HF TrainerState (gives the optimizer-step clock)."""
421
+ try:
422
+ import gc
423
+ for o in gc.get_objects():
424
+ if (type(o).__name__ == "TrainerState"
425
+ and int(getattr(o, "max_steps", 0) or 0) > 0):
426
+ return o
427
+ except Exception:
428
+ pass
429
+ return None
430
+
431
+ def k_for(self, step):
432
+ """Geometric interpolation K_START -> K_END, clamped to the endpoints."""
433
+ p = min(1.0, max(0.0, step / float(self.steps)))
434
+ k = self.k_start * (self.k_end / float(self.k_start)) ** p
435
+ lo, hi = min(self.k_start, self.k_end), max(self.k_start, self.k_end)
436
+ return max(lo, min(hi, int(round(k))))
437
+
438
+ def advance(self, sid, training):
439
+ """Refresh k once per micro-batch; skip gradient-checkpoint recompute."""
440
+ if not training or sid != self._leader:
441
+ return
442
+ if torch._C._current_graph_task_id() != -1: # inside backward
443
+ return
444
+ self._micro += 1
445
+ if self._state is None and self._micro in self._PROBE_AT:
446
+ self._state = self._probe_state()
447
+ self._step = (int(self._state.global_step) if self._state is not None
448
+ else self._micro)
449
+ self._k = self.k_for(self._step)
450
+ if not self._logged:
451
+ self._logged = True
452
+ print(f"[DASH k-sched] hard top-k: {self.k_start} -> {self.k_end} "
453
+ f"geometric over {self.steps} steps "
454
+ f"(clock={'TrainerState' if self._state is not None else 'micro-batch'})",
455
+ flush=True)
456
+ elif self._micro % self.log_every == 0:
457
+ print(f"[DASH k-sched] step={self._step} k={self._k}", flush=True)
458
+
459
+ @property
460
+ def k(self):
461
+ return self._k
462
+
463
+
464
+ _KS = _KSchedule()
465
+
466
+
467
+ class _PerLayerScheduler:
468
+ """Per-layer entmax temperature, closed-looped onto the CURRENT k budget.
469
+
470
+ Variant B keeps entmax (adaptive, differentiable sparse prior) but the
471
+ attended set is a hard top-k, so entmax no longer decides HOW MANY blocks are
472
+ read -- it decides how the kept blocks are WEIGHTED, and therefore how much
473
+ gradient each one carries. Two failure modes follow, both per-layer:
474
+
475
+ * support << k -> most of the k kept blocks have prior 0, their bias clamps
476
+ to log(prior_floor) and they contribute no gradient.
477
+ * support >> k -> the prior is flat over the kept blocks, so the bias
478
+ carries almost no ranking information.
479
+
480
+ So each layer runs its OWN loop driving its entmax support size onto the
481
+ current k (``_KS.k``, which itself anneals 64->12). The base overlay shared
482
+ ONE global scale and could only pin the across-layer MEAN: the per-layer
483
+ counts spread 13-27 at a mean of 16, i.e. some layers saturated while others
484
+ stayed flat. ``prior_floor`` follows each layer's own scale geometrically.
485
+
486
+ The scale MUST be constant for a whole micro-batch: with gradient
487
+ checkpointing the layers are recomputed in backward, and a scale that moved
488
+ mid-forward changes the entmax support -> a different bias -> the recompute
489
+ disagrees with the saved forward. So every layer refreshes its scale at most
490
+ once per micro-batch, from the PREVIOUS micro-batch's measurement.
491
+ """
492
+
493
+ # gc scans are not free, so only probe for the TrainerState a few times.
494
+ _PROBE_AT = (1, 4, 16, 64, 256)
495
+
496
+ def __init__(self):
497
+ self.on = os.environ.get("DASH_SPARSITY_SCHED", "0") == "1"
498
+ self.scale_max = _envf("DASH_SCHED_SCALE_MAX", 128.0)
499
+ self.scale_min = _envf("DASH_SCHED_SCALE_MIN", 0.05)
500
+ self.log_every = max(1, _envi("DASH_SCHED_LOG_EVERY", 50))
501
+ self.gain = _envf("DASH_SCHED_GAIN", 0.1) # log-domain P gain
502
+ self.ema = _envf("DASH_SCHED_EMA", 0.9) # measurement smoothing
503
+ self.max_step_ratio = _envf("DASH_SCHED_MAX_STEP", 1.05) # per-step cap
504
+ # one-shot PER-LAYER calibration: bisect this layer's scale on its first
505
+ # real batch so it starts exactly at the initial k, instead of inheriting
506
+ # a temperature tuned for another layer's score spread.
507
+ self.calibrate_on = _envi("DASH_SCHED_CALIB", 1) == 1
508
+ self.calib_rows = max(16, _envi("DASH_SCHED_CALIB_ROWS", 512))
509
+ self.floor_start = None # resolved from config on first register()
510
+ self.floor_end = _envf("DASH_SCHED_FLOOR_END", 1.0e-4)
511
+ self.scale_init = None
512
+ self.scale_ref = None # floor interpolates over [init, ref]
513
+ self._scale = {} # sid -> entmax scale
514
+ self._floor = {} # sid -> prior floor
515
+ self._measured = {} # sid -> EMA of entmax-active blocks/query
516
+ self._pending = {} # sid -> newest raw measurement
517
+ self._micro = {} # sid -> own micro-batch counter
518
+ self._calibrated = set()
519
+ self._step = 0
520
+ self._state = None
521
+ self._leader = None
522
+ self._n_reg = 0
523
+ self._logged = False
524
+ # -- setup ---------------------------------------------------------------
525
+ def register(self, cfg_scale, cfg_floor):
526
+ """Returns an id; the first caller becomes the clock leader."""
527
+ if self.scale_init is None:
528
+ self.scale_init = _envf("DASH_SCHED_SCALE_START", cfg_scale)
529
+ self.scale_ref = _envf("DASH_SCHED_SCALE_END", 8.0)
530
+ self.floor_start = _envf("DASH_SCHED_FLOOR_START", cfg_floor)
531
+ sid = self._n_reg
532
+ self._n_reg += 1
533
+ if self._leader is None:
534
+ self._leader = sid
535
+ self._scale[sid] = self.scale_init
536
+ self._floor[sid] = self.floor_start
537
+ self._micro[sid] = 0
538
+ return sid
539
+
540
+ # -- clock ---------------------------------------------------------------
541
+ def _probe_state(self):
542
+ """Locate the live HF TrainerState (gives the optimizer-step clock)."""
543
+ try:
544
+ import gc
545
+ for o in gc.get_objects():
546
+ if (type(o).__name__ == "TrainerState"
547
+ and int(getattr(o, "max_steps", 0) or 0) > 0):
548
+ return o
549
+ except Exception:
550
+ pass
551
+ return None
552
+
553
+ def advance(self, sid, training):
554
+ """Refresh THIS layer's scale; skip gradient-checkpoint recompute."""
555
+ if not self.on or not training:
556
+ return
557
+ if torch._C._current_graph_task_id() != -1: # inside backward
558
+ return
559
+ n = self._micro.get(sid, 0) + 1
560
+ self._micro[sid] = n
561
+ if sid == self._leader:
562
+ if self._state is None and n in self._PROBE_AT:
563
+ self._state = self._probe_state()
564
+ self._step = (int(self._state.global_step) if self._state is not None
565
+ else n)
566
+ # fold the previous micro-batch's measurement in, then P-control
567
+ m = self._pending.pop(sid, None)
568
+ if m is not None:
569
+ prev = self._measured.get(sid)
570
+ self._measured[sid] = m if prev is None else (
571
+ self.ema * prev + (1.0 - self.ema) * m)
572
+ self._scale[sid] = self._closed_loop_scale(sid)
573
+ self._floor[sid] = self._floor_for(self._scale[sid])
574
+ if not self._logged:
575
+ self._logged = True
576
+ print(f"[DASH sched] per-layer closed loop ON: setpoint = current k "
577
+ f"(anneal {_KS.k_start}->{_KS.k_end}), scale init "
578
+ f"{self.scale_init:g} in [{self.scale_min:g}, {self.scale_max:g}], "
579
+ f"floor {self.floor_start:g}->{self.floor_end:g}, gain={self.gain:g} "
580
+ f"ema={self.ema:g} max_step={self.max_step_ratio:g}", flush=True)
581
+ elif sid == self._leader and n % self.log_every == 0:
582
+ print(f"[DASH sched] step={self._step} k={_KS.k}" + self._table(),
583
+ flush=True)
584
+
585
+ def _table(self):
586
+ """`| L0 scale=.. act=.. | L1 ...` -- every layer's temperature + support."""
587
+ out = []
588
+ for s in sorted(self._scale):
589
+ act = self._measured.get(s)
590
+ out.append(f" | L{s} scale={self._scale[s]:.2f} "
591
+ f"act={'-' if act is None else format(act, '.1f')}")
592
+ return "".join(out)
593
+ # -- closed loop ---------------------------------------------------------
594
+ def _floor_for(self, scale):
595
+ """Tie the floor to how far the scale has travelled (log domain).
596
+
597
+ WHY the floor must follow: the router's gradient channel is
598
+ ``bias = (log(clamp(p, floor)) - mu)/sigma``. If the active p_j fall below
599
+ `floor`, every logp clamps to the same value, the mean-centered bias
600
+ collapses to 0 and the router stops learning. A sharper prior has smaller
601
+ active p, so the floor is annealed DOWN with the scale.
602
+ """
603
+ a, b = self.scale_init, self.scale_ref
604
+ if not a or b <= a:
605
+ p = 0.0
606
+ else:
607
+ p = min(1.0, max(0.0, math.log(max(scale, 1e-8) / a) / math.log(b / a)))
608
+ fa, fb = self.floor_start, self.floor_end
609
+ return fa * (fb / max(fa, 1e-12)) ** p # geometric, floor goes DOWN
610
+
611
+ def _closed_loop_scale(self, sid):
612
+ """Multiplicative P-control in the log domain, per layer: support above
613
+ the k budget -> raise the scale (sharper entmax), below -> back off."""
614
+ tgt = float(_KS.k)
615
+ m = self._measured.get(sid)
616
+ cur = self._scale[sid]
617
+ if tgt <= 0 or m is None:
618
+ return cur
619
+ ratio = math.exp(self.gain * math.log(max(m, 1e-6) / tgt))
620
+ ratio = min(self.max_step_ratio, max(1.0 / self.max_step_ratio, ratio))
621
+ return min(self.scale_max, max(self.scale_min, cur * ratio))
622
+
623
+ def observe(self, sid, prior, cand, training):
624
+ """Record this layer's entmax support size for its own loop.
625
+
626
+ MEASUREMENT ONLY -- the scale it feeds is applied by ``advance()`` at the
627
+ next micro-batch boundary (mutating it mid-forward breaks the
628
+ gradient-checkpoint recompute).
629
+ """
630
+ if not self.on or not training:
631
+ return
632
+ if torch._C._current_graph_task_id() != -1:
633
+ return
634
+ with torch.no_grad():
635
+ a = ((prior > 0) & cand).sum(dim=-1).float().mean()
636
+ try:
637
+ import torch.distributed as dist
638
+ if dist.is_available() and dist.is_initialized():
639
+ dist.all_reduce(a, op=dist.ReduceOp.AVG) # keep ranks in sync
640
+ except Exception:
641
+ pass
642
+ m = float(a.item())
643
+ if math.isfinite(m):
644
+ self._pending[sid] = m
645
+
646
+ def needs_calib(self, sid, training):
647
+ return (self.on and self.calibrate_on and sid not in self._calibrated
648
+ and training and torch._C._current_graph_task_id() == -1)
649
+
650
+ @torch.no_grad()
651
+ def calibrate(self, sid, block_scores, cand, neg, sqrt_d):
652
+ """Bisect THIS layer's entmax scale on its first batch so its support
653
+ starts at the current k budget.
654
+
655
+ The active-block count is monotonically non-increasing in the scale, so a
656
+ geometric bisection converges in a few iterations. Runs on a strided
657
+ subset of the query rows (the full [B,S,nblk] would sort tens of millions
658
+ of entries per probe).
659
+ """
660
+ self._calibrated.add(sid) # never retry, even if this bails out
661
+ target = float(_KS.k)
662
+ stride = max(1, block_scores.shape[1] // self.calib_rows)
663
+ bs = block_scores[:, ::stride].float()
664
+ cd = cand[:, ::stride]
665
+
666
+ def active(s):
667
+ x = torch.where(cd, bs * (s / sqrt_d), bs.new_full((), neg))
668
+ return float(((entmax15(x, dim=-1) > 0) & cd).sum(dim=-1).float().mean())
669
+
670
+ lo, hi = self.scale_min, self.scale_max
671
+ if active(lo) <= target:
672
+ s, why = lo, f"already at {active(lo):.1f} blk <= k"
673
+ elif active(hi) > target:
674
+ s, why = hi, f"SCALE_MAX only reaches {active(hi):.1f} blk"
675
+ else:
676
+ for _ in range(20):
677
+ mid = math.sqrt(lo * hi)
678
+ if active(mid) > target:
679
+ lo = mid
680
+ else:
681
+ hi = mid
682
+ s, why = hi, f"bisected, {active(hi):.1f} blk"
683
+ try:
684
+ import torch.distributed as dist
685
+ if dist.is_available() and dist.is_initialized():
686
+ t = torch.tensor([s], device=block_scores.device, dtype=torch.float32)
687
+ dist.all_reduce(t, op=dist.ReduceOp.AVG) # one scale per layer, all ranks
688
+ s = float(t.item())
689
+ except Exception:
690
+ pass
691
+ self._scale[sid] = s
692
+ self._floor[sid] = self._floor_for(s)
693
+ print(f"[DASH sched] L{sid} calibrated scale={s:.3f} for k={target:g} "
694
+ f"({why})", flush=True)
695
+
696
+ # -- readout -------------------------------------------------------------
697
+ def scale(self, sid, fallback):
698
+ return self._scale.get(sid, fallback) if self.on else fallback
699
+
700
+ def floor(self, sid, fallback):
701
+ return self._floor.get(sid, fallback) if self.on else fallback
702
+
703
+ def state_str(self, sid):
704
+ if not self.on or sid not in self._scale:
705
+ return ""
706
+ return (f" [sched step={self._step} scale={self._scale[sid]:.3f} "
707
+ f"floor={self._floor[sid]:.2e}]")
708
+
709
+
710
+ _SCHED = _PerLayerScheduler()
711
+
712
+
713
+ def _dbg_finite(name, t):
714
+ """Forward-side finiteness probe (only under DASH_DEBUG_ANOMALY=1)."""
715
+ if _DASH_DEBUG and torch.is_tensor(t) and not torch.isfinite(t).all():
716
+ print(f"[DASH-DBG fwd] non-finite in {name}: shape={tuple(t.shape)} "
717
+ f"nan={int(torch.isnan(t).sum())} inf={int(torch.isinf(t).sum())}",
718
+ flush=True)
719
+
720
+
721
+ def _dbg_grad(name, t):
722
+ """Register a backward hook that reports the first non-finite gradient."""
723
+ if _DASH_DEBUG and torch.is_tensor(t) and t.requires_grad:
724
+ def _hook(g, _n=name):
725
+ if g is not None and not torch.isfinite(g).all():
726
+ print(f"[DASH-DBG bwd] non-finite GRAD at {_n}: shape={tuple(g.shape)} "
727
+ f"nan={int(torch.isnan(g).sum())} inf={int(torch.isinf(g).sum())}",
728
+ flush=True)
729
+ return None
730
+ t.register_hook(_hook)
731
+
732
+ # log-floor used for zeroed entmax priors; `base > _LOG_TINY + 1` <=> prior > 0.
733
+ _LOG_TINY = math.log(1e-20)
734
+ # finite "-inf" for SDPA additive masks (real -inf makes SDPA's fused backward
735
+ # produce NaN gradients via 0*inf). exp(_MASK_NEG - rowmax) underflows to 0.
736
+ _MASK_NEG = -1.0e9
737
+ # The router NEVER uses real -inf: masked_fill(-inf) + amax makes fully-masked
738
+ # rows -inf, and that -inf reaches the amax/matmul backward, which manufactures
739
+ # inf/NaN grads (0*inf) in q_norm/k_norm -> the MulBackward0 NaN anomaly.
740
+ # A finite sentinel keeps every backward well defined.
741
+ _SCORE_NEG = -1.0e9
742
+ # fp32 overflow guard: a *finite but huge* grad (e.g. 1e37) still overflows the
743
+ # grad_weight reduction inside RMSNorm's mul backward (sum -> +-inf -> NaN), so
744
+ # grads must be BOUNDED, not merely nan_to_num'd.
745
+ _GRAD_CLAMP = 1.0e4
746
+ _ACT_CLAMP = 1.0e4
747
+
748
+
749
+ def _sanitize_grad(g):
750
+ """nan_to_num + bound: keeps every downstream reduction inside fp32 range."""
751
+ if g is None:
752
+ return None
753
+ return torch.nan_to_num(g, nan=0.0, posinf=0.0, neginf=0.0).clamp(
754
+ -_GRAD_CLAMP, _GRAD_CLAMP)
755
+
756
+
757
+ # ---------------------------------------------------------------------------
758
+ # 1.5-entmax (Peters, Niculae & Martins, 2019). Exact forward + backward.
759
+ # The ONLY custom autograd in this file. Operates over `dim` (the block axis).
760
+ # Masked-out entries must be passed as a finite, very-negative value (not -inf)
761
+ # so the sorting-based threshold stays numerically well-defined; they receive
762
+ # probability exactly 0.
763
+ # ---------------------------------------------------------------------------
764
+ def _make_ix_like(X, dim):
765
+ d = X.size(dim)
766
+ rho = torch.arange(1, d + 1, device=X.device, dtype=X.dtype)
767
+ view = [1] * X.dim()
768
+ view[0] = -1
769
+ return rho.view(view).transpose(0, dim)
770
+
771
+
772
+ def _entmax_threshold_and_support(X, dim=-1):
773
+ Xsrt, _ = torch.sort(X, descending=True, dim=dim)
774
+ rho = _make_ix_like(X, dim)
775
+ mean = Xsrt.cumsum(dim) / rho
776
+ mean_sq = (Xsrt ** 2).cumsum(dim) / rho
777
+ ss = rho * (mean_sq - mean ** 2)
778
+ delta = (1 - ss) / rho
779
+ delta_nz = torch.clamp(delta, 0)
780
+ tau = mean - torch.sqrt(delta_nz)
781
+ support_size = (tau <= Xsrt).sum(dim).unsqueeze(dim)
782
+ # Guard: a degenerate/NaN row can give support_size == 0, so support_size - 1
783
+ # would be -1 and crash `gather` with a device-side index-out-of-bounds
784
+ # assert. Clamp to >=1 (falls back to the top element's threshold).
785
+ support_size = support_size.clamp(min=1)
786
+ tau_star = tau.gather(dim, support_size - 1)
787
+ return tau_star, support_size
788
+
789
+
790
+ class _Entmax15(torch.autograd.Function):
791
+ @staticmethod
792
+ def forward(ctx, X, dim):
793
+ ctx.dim = dim
794
+ max_val, _ = X.max(dim=dim, keepdim=True)
795
+ X = X - max_val # numerical stability, shift-invariant
796
+ X = X / 2 # exponent handling for alpha=1.5
797
+ tau_star, _ = _entmax_threshold_and_support(X, dim)
798
+ Y = torch.clamp(X - tau_star, min=0) ** 2
799
+ ctx.save_for_backward(Y)
800
+ return Y
801
+
802
+ @staticmethod
803
+ def backward(ctx, dY):
804
+ (Y,) = ctx.saved_tensors
805
+ gppr = Y.sqrt() # = clamp(X - tau, 0)
806
+ dX = dY * gppr
807
+ q = dX.sum(ctx.dim) / gppr.sum(ctx.dim).clamp_min(1e-12)
808
+ q = q.unsqueeze(ctx.dim)
809
+ dX = dX - q * gppr
810
+ return dX, None
811
+
812
+
813
+ def entmax15(X, dim=-1):
814
+ return _Entmax15.apply(X, dim)
815
+
816
+
817
+ # RoPE for the index projections (available at runtime in the assembled model
818
+ # dir; absent in a bare unit-test import -> rope is skipped).
819
+ try:
820
+ from ._sparse_base_impl import _llama_apply_rotary_pos_emb as _rope_fn
821
+ except Exception:
822
+ _rope_fn = None
823
+
824
+ # The router's norms are NOT in the checkpoint, so they are "missing keys" and get
825
+ # whatever `_init_weights` gives them. That hook only matches nn.Linear /
826
+ # nn.Embedding / _GemmaRMSNorm -- an unmatched module type is left as
827
+ # UNINITIALIZED memory, which is what produced `WEIGHT non-finite: nan=1` in the
828
+ # very first forward. Subclassing _GemmaRMSNorm makes the isinstance check hit,
829
+ # so the weight is zero-init'd (Gemma scale = 1 + weight => scale 1 at init).
830
+ try:
831
+ from ._sparse_base_impl import _GemmaRMSNorm as _RMSNormBase
832
+ except Exception:
833
+ class _RMSNormBase(nn.Module):
834
+ def __init__(self, dim, eps=1e-6):
835
+ super().__init__()
836
+ self.eps = eps
837
+ self.weight = nn.Parameter(torch.zeros(dim))
838
+
839
+
840
+ class _RMSNorm(_RMSNormBase):
841
+ """Gemma-style RMSNorm (scale = 1 + weight, weight init 0) with fp32 +
842
+ finiteness guards for the router's index projections."""
843
+
844
+ def forward(self, x):
845
+ x32 = x.float()
846
+ # Sanitize BEFORE the variance reduction: a non-finite input makes
847
+ # mean(x^2)=inf -> rsqrt(inf)=0 -> inf*0=NaN, and that NaN is baked into
848
+ # the tensors saved for backward, so a post-hoc nan_to_num on the output
849
+ # cannot undo it (this is exactly the MulBackward0 NaN anomaly mode caught).
850
+ # Zeroing non-finite positions here keeps BOTH forward and backward finite.
851
+ x32 = torch.nan_to_num(x32, nan=0.0, posinf=0.0, neginf=0.0)
852
+ x32 = x32 * torch.rsqrt(x32.pow(2).mean(-1, keepdim=True) + self.eps)
853
+ x32 = x32.clamp(-_ACT_CLAMP, _ACT_CLAMP)
854
+ out = (x32 * (1.0 + self.weight.float())).clamp(
855
+ -_ACT_CLAMP, _ACT_CLAMP).to(x.dtype)
856
+ # The NaN anomaly fires INSIDE this mul's backward (grad wrt `weight`):
857
+ # incoming grads that are finite-but-huge overflow the sum over [B,S] to
858
+ # +-inf and cancel to NaN. Downstream hooks (idx_q/idx_k, post-RoPE)
859
+ # cannot prevent that, so bound the grad AT this output.
860
+ if out.requires_grad:
861
+ out.register_hook(_sanitize_grad)
862
+ if _DASH_DEBUG:
863
+ if not torch.isfinite(self.weight).all():
864
+ print(f"[DASH-DBG rmsnorm] WEIGHT non-finite: "
865
+ f"nan={int(torch.isnan(self.weight).sum())} "
866
+ f"inf={int(torch.isinf(self.weight).sum())}", flush=True)
867
+ _dbg_finite("rmsnorm_out", out)
868
+ _dbg_grad("rmsnorm_out(grad_in)", out)
869
+ return out
870
+
871
+
872
+ # ---------------------------------------------------------------------------
873
+ # DASH router / attention module
874
+ # ---------------------------------------------------------------------------
875
+ class DashAttention(nn.Module):
876
+ """Differentiable adaptive sparse hierarchical attention.
877
+
878
+ Block scoring uses a SEPARATE low-dim index projection of ``hidden_states``
879
+ (the proven-stable Lightning-Indexer path that ``e2e.sh`` uses), NOT a
880
+ softmax summary over the raw attention keys. The raw-key summary averaged
881
+ over EVERY key position, so any non-finite key from the base model (dead /
882
+ special-token positions the sparse attention never reads -- which is why
883
+ sparse.sh is unaffected) turned a whole block's summary into NaN and poisoned
884
+ the router matmul's backward. The index projection avoids that entirely.
885
+
886
+ On top of the (non-local, causal) block-max scores we apply 1.5-entmax to get
887
+ a sparse, differentiable ``prior``; its top-k gives the block selection and
888
+ ``log(prior)`` is fed as the differentiable per-block bias to the fused e2e
889
+ kernel -> router trains end-to-end from the LM loss, no KL distillation.
890
+ """
891
+
892
+ def __init__(self, config):
893
+ super().__init__()
894
+ sp = getattr(config, "sparse_attention_config", {}) or {}
895
+ self.num_heads = config.num_attention_heads
896
+ self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
897
+
898
+ # block granularity reuses the base sparse knobs so configs stay aligned
899
+ self.block_size = int(sp.get("sparse_block_size", 128))
900
+ self.local_blocks = max(1, int(sp.get("sparse_local_block", 1)))
901
+ self.local_window = self.local_blocks * self.block_size
902
+
903
+ # DASH-specific knobs
904
+ self.scaling_factor = float(sp.get("dash_scaling_factor", 2.0)) # entmax sharpness
905
+ self.q_chunk = int(sp.get("dash_query_chunk", 2048)) # memory knob
906
+ self.image_only = bool(sp.get("dash_image_only", True))
907
+ self._image_token_id = int(sp.get("image_token_id", 128815))
908
+ # log-prior bias = log(prior)/sigma; prior_floor bounds the 1/p gradient.
909
+ self.sigma = float(sp.get("dash_sigma", 1.0))
910
+ self._prior_floor = float(sp.get("dash_prior_floor", 1e-2))
911
+
912
+ self.topk_blocks = int(sp.get("sparse_topk_blocks", 16))
913
+ self.remote_blocks = max(1, self.topk_blocks - self.local_blocks)
914
+ self.use_triton = bool(sp.get("dash_use_triton", True))
915
+
916
+ # single-head index projections (like DeepseekV2Indexer, minus KL).
917
+ self.index_dim = int(sp.get("sparse_index_dim", self.head_dim))
918
+ eps = getattr(config, "rms_norm_eps", 1e-6)
919
+ self.q_proj = nn.Linear(config.hidden_size, self.index_dim, bias=False)
920
+ self.k_proj = nn.Linear(config.hidden_size, self.index_dim, bias=False)
921
+ self.q_norm = _RMSNorm(self.index_dim, eps=eps)
922
+ self.k_norm = _RMSNorm(self.index_dim, eps=eps)
923
+
924
+ # Stage-0 summarization mode (env DASH_STAGE0):
925
+ # "indexer" (default) -> block-max of token-level idx_q.idx_k (current).
926
+ # "head_cls" -> paper Stage-0: a learned per-router summary
927
+ # query `head_cls` (zero-init => starts as mean
928
+ # pooling) attends the index-keys within each
929
+ # chunk to build one summary key k_bar_c, then
930
+ # the block score is idx_q . k_bar_c.
931
+ self.stage0 = os.environ.get("DASH_STAGE0", "indexer").lower()
932
+ # Learned summary query (zero-init: uniform local attention == mean pool
933
+ # at the start, then gradually learns an expressive chunk summary). Kept
934
+ # unconditionally so a checkpoint can switch modes without a shape change.
935
+ self.head_cls = nn.Parameter(torch.zeros(self.index_dim))
936
+
937
+ self._scale = 1.0 / math.sqrt(self.head_dim) # main attention
938
+ self._idx_scale = self.scaling_factor / math.sqrt(self.index_dim) # entmax input
939
+ self._neg = -1.0e4 # finite mask fill for entmax candidate masking
940
+
941
+ # Per-layer entmax temperature loop + the shared k schedule. Both are
942
+ # registered here so this layer gets its own scale/floor slot and its own
943
+ # ordinal (the L<sid> tag in every log line).
944
+ self._sched_id = _SCHED.register(self.scaling_factor, self._prior_floor)
945
+ _KS.register()
946
+
947
+ # Live handle on config.sparse_attention_config: the ROUTER STATE that
948
+ # inference must reproduce (entmax scale, prior floor, stage-0 mode) is
949
+ # partly runtime (the scheduler anneals scale/floor) and partly env
950
+ # (DASH_STAGE0). save_pretrained serializes this dict, so mirroring the
951
+ # current values into it is what makes the exported checkpoint
952
+ # self-describing -- otherwise inference re-reads the ORIGINAL
953
+ # dash_scaling_factor / dash_prior_floor and routes at a completely
954
+ # different temperature than the trained one.
955
+ # NOTE: the scale/floor are PER LAYER now, so the scalar keys carry the
956
+ # leader layer's values (backwards compatible) and the full profile goes
957
+ # to dash_scale_per_layer / dash_floor_per_layer. dash_topk_blocks is the
958
+ # k this checkpoint was trained at -- mid-anneal exports are NOT at K_END.
959
+ self._sp = sp if isinstance(sp, dict) else None
960
+ self._persist_router_cfg()
961
+
962
+ # -- export the effective router state into the saved config -------------
963
+ def _persist_router_cfg(self):
964
+ if self._sp is None or self._sched_id != _SCHED._leader:
965
+ return
966
+ self._sp["dash_stage0"] = self.stage0
967
+ self._sp["dash_router"] = "entmax_topk_perlayer"
968
+ self._sp["dash_sigma"] = float(self.sigma)
969
+ self._sp["dash_topk_blocks"] = int(_KS.k)
970
+ self._sp["dash_scaling_factor"] = float(
971
+ self._cur_idx_scale() * math.sqrt(self.index_dim))
972
+ self._sp["dash_prior_floor"] = float(self._cur_prior_floor())
973
+ self._sp["dash_scale_per_layer"] = [
974
+ float(_SCHED.scale(s, self.scaling_factor)) for s in sorted(_SCHED._scale)]
975
+ self._sp["dash_floor_per_layer"] = [
976
+ float(_SCHED.floor(s, self._prior_floor)) for s in sorted(_SCHED._floor)]
977
+
978
+ # -- scheduled knobs (identity when the scheduler is off) ---------------
979
+ def _cur_idx_scale(self):
980
+ return (_SCHED.scale(self._sched_id, self.scaling_factor)
981
+ / math.sqrt(self.index_dim))
982
+
983
+ def _cur_prior_floor(self):
984
+ return _SCHED.floor(self._sched_id, self._prior_floor)
985
+
986
+ @torch.no_grad()
987
+ def build_image_block_mask(self, source_input_ids, key_length, nblk, device):
988
+ """Return [B, nblk] bool: block contains >=1 image token. None if n/a."""
989
+ if source_input_ids is None or source_input_ids.ndim != 2:
990
+ return None
991
+ ids = source_input_ids[..., :key_length].to(device)
992
+ if ids.shape[-1] < key_length:
993
+ ids = F.pad(ids, (0, key_length - ids.shape[-1]), value=-1)
994
+ img = ids == self._image_token_id # [B, key_length]
995
+ if img.sum() == 0:
996
+ return None
997
+ Sp = nblk * self.block_size
998
+ if Sp > img.shape[-1]:
999
+ img = F.pad(img, (0, Sp - img.shape[-1]), value=False)
1000
+ else:
1001
+ img = img[..., :Sp]
1002
+ return img.view(img.shape[0], nblk, self.block_size).any(dim=-1) # [B,nblk]
1003
+
1004
+ # -- block scoring (indexer-style, non-local causal block-max) --------
1005
+ def _block_scores(self, hidden_states, position_embeddings, position_ids, nblk):
1006
+ """[B,S,nblk] fp32 remote block scores; ``_SCORE_NEG`` where a block has no
1007
+ non-local causal key. Computed from a low-dim index projection (finite by
1008
+ construction), chunked over queries to bound memory."""
1009
+ B, S, _ = hidden_states.shape
1010
+ device = hidden_states.device
1011
+ # Router reads hidden_states directly; if an upstream layer emitted a
1012
+ # non-finite activation, the projections + matmul below manufacture NaN
1013
+ # block_scores (and NaN grads in q_norm/k_norm). Sanitize the router's own
1014
+ # view so it never propagates upstream NaN into the routing path.
1015
+ hs = torch.nan_to_num(hidden_states, nan=0.0, posinf=0.0, neginf=0.0)
1016
+ idx_q = self.q_norm(self.q_proj(hs).view(B, S, 1, self.index_dim)).transpose(1, 2)
1017
+ idx_k = self.k_norm(self.k_proj(hs).view(B, S, 1, self.index_dim)).transpose(1, 2)
1018
+ if _rope_fn is not None and position_embeddings is not None:
1019
+ cos, sin = position_embeddings
1020
+ idx_q, idx_k = _rope_fn(idx_q, idx_k, cos[..., :self.index_dim], sin[..., :self.index_dim])
1021
+ # nan_to_num defaults map +/-inf to +/-3.4e38, which then OVERFLOWS the
1022
+ # score matmul back into inf -> NaN. Map non-finite to 0 instead.
1023
+ idx_q = torch.nan_to_num(idx_q[:, 0].float(), nan=0.0, posinf=0.0, neginf=0.0) # [B,S,d]
1024
+ idx_k = torch.nan_to_num(idx_k[:, 0].float(), nan=0.0, posinf=0.0, neginf=0.0) # [B,S,d]
1025
+ # The amax/matmul backward below sits BETWEEN block_scores and the q/k_norm
1026
+ # RMSNorms and can manufacture NaN from the -inf entries masked_fill'd into
1027
+ # `sc` even when the incoming block_scores grad is finite -- which is why a
1028
+ # block_scores-level hook does not stop the k_norm MulBackward0 NaN. Report
1029
+ # (debug) then sanitize the grad right at the norm outputs (tight guard).
1030
+ _dbg_grad("idx_q(grad_in)", idx_q); _dbg_grad("idx_k(grad_in)", idx_k)
1031
+ if idx_q.requires_grad:
1032
+ idx_q.register_hook(_sanitize_grad)
1033
+ if idx_k.requires_grad:
1034
+ idx_k.register_hook(_sanitize_grad)
1035
+ kpos = torch.arange(S, device=device)
1036
+ Sp = nblk * self.block_size
1037
+ out = []
1038
+ for c0 in range(0, S, self.q_chunk):
1039
+ c1 = min(c0 + self.q_chunk, S)
1040
+ sc = torch.matmul(idx_q[:, c0:c1], idx_k.transpose(-1, -2)) # [B,Lq,S]
1041
+ sc = sc.clamp(-_ACT_CLAMP, _ACT_CLAMP)
1042
+ pos = position_ids[:, c0:c1] # [B,Lq]
1043
+ future = kpos[None, None, :] > pos[:, :, None]
1044
+ local = (kpos[None, None, :] > (pos[:, :, None] - self.local_window)) & ~future
1045
+ sc = sc.masked_fill(future | local, _SCORE_NEG) # remote-only
1046
+ if Sp > S:
1047
+ sc = F.pad(sc, (0, Sp - S), value=_SCORE_NEG)
1048
+ sc = sc.view(B, c1 - c0, nblk, self.block_size)
1049
+ out.append(sc.amax(dim=-1)) # [B,Lq,nblk]
1050
+ return torch.cat(out, dim=1) # [B,S,nblk]
1051
+
1052
+ # -- Stage-0 (head_cls): learned-summary block scoring (paper) --------
1053
+ def _block_scores_headcls(self, hidden_states, position_embeddings, position_ids, nblk):
1054
+ """[B,S,nblk] fp32 remote block scores via a learned chunk summary.
1055
+
1056
+ Mirrors the paper's Stage-0/Stage-1: a per-router summary query
1057
+ ``head_cls`` (zero-init -> starts as mean pooling) attends the index-keys
1058
+ within each chunk to form one summary key ``k_bar_c``; the block score is
1059
+ ``idx_q_i . k_bar_c``. Same single-head index space (q_proj/k_proj/norms)
1060
+ and the same remote-only (causal minus local-window) masking as the
1061
+ ``indexer`` path, so it drops straight into ``_route``. NOTE: the summary
1062
+ is over the INDEX-key space (not the main keys) to stay single-head and
1063
+ self-contained -- a deliberate adaptation of the paper to this codebase.
1064
+ """
1065
+ B, S, _ = hidden_states.shape
1066
+ device = hidden_states.device
1067
+ hs = torch.nan_to_num(hidden_states, nan=0.0, posinf=0.0, neginf=0.0)
1068
+ idx_q = self.q_norm(self.q_proj(hs).view(B, S, 1, self.index_dim)).transpose(1, 2)
1069
+ idx_k = self.k_norm(self.k_proj(hs).view(B, S, 1, self.index_dim)).transpose(1, 2)
1070
+ if _rope_fn is not None and position_embeddings is not None:
1071
+ cos, sin = position_embeddings
1072
+ idx_q, idx_k = _rope_fn(idx_q, idx_k, cos[..., :self.index_dim], sin[..., :self.index_dim])
1073
+ idx_q = torch.nan_to_num(idx_q[:, 0].float(), nan=0.0, posinf=0.0, neginf=0.0) # [B,S,d]
1074
+ idx_k = torch.nan_to_num(idx_k[:, 0].float(), nan=0.0, posinf=0.0, neginf=0.0) # [B,S,d]
1075
+ if idx_q.requires_grad:
1076
+ idx_q.register_hook(_sanitize_grad)
1077
+ if idx_k.requires_grad:
1078
+ idx_k.register_hook(_sanitize_grad)
1079
+ d = self.index_dim
1080
+ inv_sqrt_d = 1.0 / math.sqrt(d)
1081
+ Sp = nblk * self.block_size
1082
+
1083
+ # Pad the index-keys up to whole chunks and track which slots are real.
1084
+ if Sp > S:
1085
+ idx_k = F.pad(idx_k, (0, 0, 0, Sp - S)) # [B,Sp,d]
1086
+ real = F.pad(idx_k.new_ones(B, S, dtype=torch.bool), (0, Sp - S), value=False)
1087
+ else:
1088
+ idx_k = idx_k[:, :Sp]
1089
+ real = idx_k.new_ones(B, Sp, dtype=torch.bool)
1090
+ kk = idx_k.view(B, nblk, self.block_size, d) # [B,nblk,bs,d]
1091
+ vmask = real.view(B, nblk, self.block_size) # [B,nblk,bs]
1092
+
1093
+ # Stage 0: head_cls local attention within each chunk -> summary key.
1094
+ # NaN-safe MASKED softmax: never feed -inf to softmax (a fully-padding
1095
+ # chunk would give all -inf -> NaN in BOTH forward AND SoftmaxBackward0,
1096
+ # which nan_to_num on the output cannot repair). Instead subtract the row
1097
+ # max (finite), exp, zero out padding multiplicatively, and normalize by a
1098
+ # clamped denominator -> forward and backward stay finite. hc=0 (zero-init)
1099
+ # -> uniform weights over valid tokens == mean pooling.
1100
+ hc = self.head_cls.to(kk.dtype).view(1, 1, 1, d)
1101
+ aw = (kk * hc).sum(-1) * inv_sqrt_d # [B,nblk,bs]
1102
+ aw = aw.clamp(-_ACT_CLAMP, _ACT_CLAMP)
1103
+ # NaN-safe masked softmax. Take the row-max over VALID tokens only: if we
1104
+ # max over padding too (padding aw==0), a chunk whose valid scores are all
1105
+ # negative would be dominated by padding, its valid exp() underflow, denom
1106
+ # -> 0, and the 1/denom in the division backward explodes (-> BmmBackward0
1107
+ # NaN downstream). Valid-only max keeps >=1 entry at e==1, so denom>=1.
1108
+ _neg = torch.finfo(aw.dtype).min
1109
+ row_max = torch.where(vmask, aw, aw.new_full((), _neg)).amax(dim=-1, keepdim=True)
1110
+ row_max = torch.where(vmask.any(dim=-1, keepdim=True), row_max,
1111
+ torch.zeros_like(row_max)) # all-pad chunk -> 0
1112
+ aw = aw - row_max
1113
+ e = torch.exp(aw) * vmask.to(aw.dtype) # padding -> 0
1114
+ w = e / e.sum(dim=-1, keepdim=True).clamp_min(1e-6) # [B,nblk,bs]
1115
+ k_bar = (w.unsqueeze(-1) * kk).sum(dim=2) # [B,nblk,d]
1116
+ # k_bar is a saved input of the Stage-1 matmul's backward -- force it
1117
+ # finite so BmmBackward0 can never emit NaN from a bad saved activation.
1118
+ k_bar = torch.nan_to_num(k_bar, nan=0.0, posinf=0.0, neginf=0.0)
1119
+ chunk_valid = vmask.any(dim=-1) # [B,nblk]
1120
+
1121
+ # Stage 1: z[i,c] = idx_q_i . k_bar_c, remote-only (causal minus local).
1122
+ blk_ids = torch.arange(nblk, device=device)
1123
+ out = []
1124
+ for c0 in range(0, S, self.q_chunk):
1125
+ c1 = min(c0 + self.q_chunk, S)
1126
+ z = torch.matmul(idx_q[:, c0:c1], k_bar.transpose(-1, -2)) # [B,Lq,nblk]
1127
+ # Sanitize the grad AT the matmul output: this is exactly the
1128
+ # grad_output BmmBackward0 consumes, so a NaN/Inf arriving from the
1129
+ # entmax/route backward (which the block_scores hook can miss under
1130
+ # gradient checkpointing) is cleaned before the matmul backward runs.
1131
+ if z.requires_grad:
1132
+ z.register_hook(_sanitize_grad)
1133
+ z = z * inv_sqrt_d
1134
+ z = z.clamp(-_ACT_CLAMP, _ACT_CLAMP)
1135
+ qblk = (position_ids[:, c0:c1] // self.block_size) # [B,Lq]
1136
+ # chunk c is a remote candidate iff it lies fully before the local
1137
+ # window: c <= qblk - local_blocks (matches the indexer path's
1138
+ # token-level `pos + local_blocks*block_size <= query_pos`).
1139
+ remote_ok = blk_ids[None, None, :] <= (qblk[:, :, None] - self.local_blocks)
1140
+ keep = remote_ok & chunk_valid[:, None, :]
1141
+ z = z.masked_fill(~keep, _SCORE_NEG)
1142
+ out.append(z)
1143
+ return torch.cat(out, dim=1) # [B,S,nblk]
1144
+
1145
+ def _route(self, block_scores, image_block_mask, block_cand=None):
1146
+ """block_scores [B,S,nblk] -> (prior [B,S,nblk], cand [B,S,nblk] bool).
1147
+
1148
+ ``block_cand`` [B,S,nblk] is the PER-QUERY candidate domain the caller builds
1149
+ from the indexer's page-aware ids (``_build_kl_candidate_mask`` -> same-page image
1150
+ bucket, with its own per-query fallback). It takes precedence over
1151
+ ``image_block_mask`` [B,nblk], which is page-agnostic (image blocks of any page,
1152
+ identical for every query) and would drop page isolation entirely.
1153
+ """
1154
+ # remote candidacy is now flagged by the finite sentinel, not by -inf
1155
+ remote = block_scores > (_SCORE_NEG * 0.5)
1156
+ cand = remote
1157
+ if block_cand is not None:
1158
+ cand = remote & block_cand
1159
+ elif self.image_only and image_block_mask is not None:
1160
+ cand = remote & image_block_mask[:, None, :]
1161
+ # per-query fallback: if image left no candidate but remote blocks exist,
1162
+ # drop the image constraint for that query.
1163
+ no_cand = ~cand.any(dim=-1, keepdim=True)
1164
+ cand = torch.where(no_cand, remote, cand)
1165
+ # one-shot per-layer calibration: land THIS layer's entmax support on the
1166
+ # current k budget, measured on real block scores (first training batch).
1167
+ if _SCHED.needs_calib(self._sched_id, self.training):
1168
+ _SCHED.calibrate(self._sched_id, block_scores.detach(), cand, self._neg,
1169
+ math.sqrt(self.index_dim))
1170
+ x = torch.where(cand, block_scores * self._cur_idx_scale(),
1171
+ block_scores.new_full((), self._neg))
1172
+ x = torch.nan_to_num(x, nan=self._neg, posinf=self._neg, neginf=self._neg)
1173
+ prior = entmax15(x, dim=-1) # [B,S,nblk]
1174
+ return prior, cand
1175
+
1176
+ def _select_and_bias(self, prior, cand, nblk):
1177
+ """Hard top-k selection + the entmax log-prior bias on the kept blocks.
1178
+
1179
+ Returns (topi [B,S,K] int64, bias [B,S,K], valid [B,S,K] bool).
1180
+
1181
+ Selection is top-k over the MASKED PRIOR; entmax is monotone in its input
1182
+ within a row, so that is the same ranking as top-k over the block scores
1183
+ and the k highest-scoring candidates are always the ones read -- including
1184
+ blocks entmax pushed to exactly 0 when its support is narrower than k.
1185
+ Those carry bias log(prior_floor) - mu: attended, but no gradient, which is
1186
+ precisely what the per-layer temperature loop exists to prevent.
1187
+
1188
+ bias = (log(clamp(p, floor)) - mu)/sigma, mu = per-query MEAN over the kept
1189
+ log-priors (paper's log_g). Mean-centering is what lets an above-average
1190
+ remote block outweigh the zero-bias local window, instead of every remote
1191
+ block being uniformly down-weighted by log(p) <= 0.
1192
+ """
1193
+ K = max(1, min(int(_KS.k), nblk))
1194
+ sel_prior = prior.masked_fill(~cand, -1.0) # non-candidates rank last
1195
+ topv, topi = sel_prior.topk(K, dim=-1) # [B,S,K]
1196
+ valid = topv >= 0.0 # a real candidate slot
1197
+ logp = torch.log(topv.clamp_min(self._cur_prior_floor()))
1198
+ cnt = valid.sum(dim=-1, keepdim=True).clamp(min=1)
1199
+ mu = torch.where(valid, logp, torch.zeros_like(logp)).sum(
1200
+ dim=-1, keepdim=True) / cnt
1201
+ bias = (logp - mu) * (1.0 / self.sigma)
1202
+ bias = torch.where(valid, bias, torch.zeros_like(bias)).clamp(
1203
+ -_ACT_CLAMP, _ACT_CLAMP)
1204
+ return topi, bias, valid
1205
+
1206
+ # -- Triton block-sparse forward (32k) --------------------------------
1207
+ def _forward_triton(self, q, k, v, topi, blk_bias, valid, S, nblk, position_ids):
1208
+ try:
1209
+ from .sparse_gather_e2e_triton import triton_gather_attention_e2e
1210
+ except Exception:
1211
+ return None
1212
+ # Fixed budget: the gathered width is exactly k for every query, every
1213
+ # layer, every step (padding slots -> index -1, skipped by the kernel).
1214
+ # A constant width also means the gradient-checkpoint recompute can never
1215
+ # disagree with the saved forward on metadata.
1216
+ block_indices = topi.masked_fill(~valid, -1).to(torch.int32).unsqueeze(1)
1217
+ out = triton_gather_attention_e2e(
1218
+ q, k, v, block_indices, blk_bias, position_ids, None,
1219
+ self.block_size, self._scale, self.local_window,
1220
+ )
1221
+ _dbg_grad("triton_out", out)
1222
+ if out is not None and not getattr(self, "_triton_logged", False):
1223
+ self._triton_logged = True
1224
+ print(f"[DASH] Triton block-sparse path active (S={S}, "
1225
+ f"block_size={self.block_size}, remote_topk={topi.shape[-1]}, "
1226
+ f"local_window={self.local_window})", flush=True)
1227
+ return out
1228
+
1229
+ # -- dense fallback (CPU / unsupported shape / validation reference) --
1230
+ def _forward_dense(self, q, k, v, topi, blk_bias, valid, S, nblk, attention_mask):
1231
+ B, H, _, D = q.shape
1232
+ device = q.device
1233
+ kidx = torch.arange(S, device=device)
1234
+ key_block = kidx // self.block_size
1235
+ # Scatter the per-selection bias back to a full [B,S,nblk] map so the dense
1236
+ # reference attends EXACTLY the same k blocks as the kernel; unselected
1237
+ # entries carry _MASK_NEG and are dropped below. `scatter` (functional)
1238
+ # keeps the autograd path into blk_bias.
1239
+ src = torch.where(valid, blk_bias, blk_bias.new_full((), _MASK_NEG))
1240
+ logp = torch.full((B, blk_bias.shape[1], nblk), _MASK_NEG,
1241
+ device=device, dtype=blk_bias.dtype).scatter(-1, topi, src)
1242
+ out_chunks = []
1243
+ for qs in range(0, S, self.q_chunk):
1244
+ qe = min(qs + self.q_chunk, S)
1245
+ qidx = torch.arange(qs, qe, device=device)
1246
+ q_c = q[:, :, qs:qe, :]
1247
+ base = logp[:, qs:qe].index_select(-1, key_block)[:, None] # [B,1,Lq,S]
1248
+ causal = (kidx[None, :] <= qidx[:, None])[None, None]
1249
+ local = ((kidx[None, :] <= qidx[:, None]) &
1250
+ (kidx[None, :] > (qidx[:, None] - self.local_window)))[None, None]
1251
+ sel = base > (_MASK_NEG * 0.5)
1252
+ keep = causal & (local | sel)
1253
+ mask = torch.where(local, torch.zeros_like(base), base)
1254
+ mask = mask.masked_fill(~keep, _MASK_NEG)
1255
+ if attention_mask is not None:
1256
+ mask = mask + attention_mask[:, :, qs:qe, :S].to(mask.dtype)
1257
+ mask = mask.clamp_min(_MASK_NEG).to(q.dtype)
1258
+ out_chunks.append(torch.nn.functional.scaled_dot_product_attention(
1259
+ q_c, k, v, attn_mask=mask, is_causal=False, scale=self._scale))
1260
+ return torch.cat(out_chunks, dim=2)
1261
+
1262
+ # -- forward -----------------------------------------------------------
1263
+ def forward(self, hidden_states, q, k, v, position_embeddings=None,
1264
+ position_ids=None, attention_mask=None, image_block_mask=None,
1265
+ block_cand=None, page_ids=None):
1266
+ """hidden_states [B,S,hidden]; q,k,v [B,H,S,D] (RoPE'd, GQA-expanded).
1267
+
1268
+ Returns [B,H,S,D] (before o_proj). image_block_mask: [B,nblk] bool or None.
1269
+ block_cand: [B,S,nblk] bool per-query candidate domain (page isolation) or None.
1270
+ page_ids: (image_token_page_ids, query_page_ids) -- used only by the
1271
+ ``DASH_LOG_PAGE_EVERY`` diagnostics, never in the computation.
1272
+ """
1273
+ B, H, S, D = q.shape
1274
+ nblk = (S + self.block_size - 1) // self.block_size
1275
+ device = q.device
1276
+ if position_ids is None:
1277
+ position_ids = torch.arange(S, device=device)[None].expand(B, -1)
1278
+
1279
+ # Refresh THIS layer's entmax temperature and the shared k (once per
1280
+ # micro-batch each; both stay frozen for the rest of the forward).
1281
+ _KS.advance(self._sched_id, self.training)
1282
+ _SCHED.advance(self._sched_id, self.training)
1283
+ # Mirror the current scale/floor/k into the config so any checkpoint saved
1284
+ # after this step carries the router state inference has to reproduce.
1285
+ self._persist_router_cfg()
1286
+
1287
+ if self.stage0 == "head_cls":
1288
+ block_scores = self._block_scores_headcls(hidden_states, position_embeddings, position_ids, nblk)
1289
+ else:
1290
+ block_scores = self._block_scores(hidden_states, position_embeddings, position_ids, nblk)
1291
+ if not getattr(self, "_stage0_logged", False):
1292
+ self._stage0_logged = True
1293
+ print(f"[DASH] Stage-0 summarization = {self.stage0!r}", flush=True)
1294
+ # Router gradient choke point. `prior`'s own hook (below) only cleans the
1295
+ # grad ENTERING prior; the entmax / where / log backward BETWEEN prior and
1296
+ # block_scores can still manufacture NaN/Inf on degenerate rows, which then
1297
+ # flows into q_norm/k_norm (the MulBackward0 anomaly that persists even
1298
+ # after the forward was made finite). Sanitizing here -- upstream of the
1299
+ # RMSNorm backward -- guarantees only finite grads reach q_proj/k_proj.
1300
+ if block_scores.requires_grad:
1301
+ block_scores.register_hook(_sanitize_grad)
1302
+ prior, cand = self._route(block_scores, image_block_mask, block_cand) # [B,S,nblk]
1303
+ topi, blk_bias, valid = self._select_and_bias(prior, cand, nblk)
1304
+ _dbg_finite("block_scores", block_scores); _dbg_finite("prior", prior)
1305
+ _dbg_grad("prior", prior)
1306
+ # `prior` is the SINGLE gradient channel from the LM loss back to the router
1307
+ # (loss -> kernel d_bias / dense mask -> blk_bias/log(prior) -> prior ->
1308
+ # block_scores -> q_proj/k_proj). The kernel's d_bias backward on our
1309
+ # log(prior) bias can emit NaN/Inf for degenerate selections (finite loss,
1310
+ # NaN grad_norm). Sanitizing the gradient AT this choke point keeps the
1311
+ # router params finite while still training them from finite gradients.
1312
+ if prior.requires_grad:
1313
+ prior.register_hook(_sanitize_grad)
1314
+
1315
+ if _LOG_SPARSITY:
1316
+ _log_train_sparsity(prior, cand, valid, self.block_size, self._sched_id)
1317
+ if (_LOG_PAGE_EVERY and page_ids is not None and block_cand is not None
1318
+ and self.training and torch._C._current_graph_task_id() == -1):
1319
+ self._page_log_n = getattr(self, "_page_log_n", 0) + 1
1320
+ if self._page_log_n % _LOG_PAGE_EVERY == 0 or self._page_log_n == 1:
1321
+ _log_page_purity(topi, valid, block_cand,
1322
+ block_scores.detach() > (_SCORE_NEG * 0.5),
1323
+ page_ids[0], page_ids[1], self.block_size, nblk,
1324
+ self._sched_id, self._page_log_n)
1325
+ # How much of the paid-for k blocks the attention really uses. Skipped in
1326
+ # the gradient-checkpoint recompute (same guard as the scheduler), else a
1327
+ # layer's counter advances twice per micro-batch.
1328
+ if _LOG_NEFF_EVERY and self.training and \
1329
+ torch._C._current_graph_task_id() == -1:
1330
+ _log_neff(q, k, topi, blk_bias, valid, position_ids, self.block_size,
1331
+ self.local_window, self._scale, self._sched_id)
1332
+ # Oracle recall of the routed set (block/score recall vs. the dense
1333
+ # attention mass). Same recompute guard: the [H,rows,S] qk is expensive
1334
+ # enough that running it twice per micro-batch is not free.
1335
+ if _LOG_RECALL_EVERY and self.training and \
1336
+ torch._C._current_graph_task_id() == -1:
1337
+ _log_oracle_recall(q, k, topi, valid, cand, position_ids,
1338
+ self.block_size, self._scale, self._sched_id)
1339
+ # this layer's own closed loop reads back its entmax support size
1340
+ _SCHED.observe(self._sched_id, prior, cand, self.training)
1341
+
1342
+ if self.use_triton and q.is_cuda and attention_mask is None:
1343
+ out = self._forward_triton(q, k, v, topi, blk_bias, valid, S, nblk,
1344
+ position_ids)
1345
+ if out is not None:
1346
+ return out
1347
+
1348
+ return self._forward_dense(q, k, v, topi, blk_bias, valid, S, nblk,
1349
+ attention_mask)
deepencoder.py ADDED
@@ -0,0 +1,1072 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch.nn as nn
2
+ import torch
3
+ import torch.nn.functional as F
4
+ import copy
5
+
6
+ from contextlib import nullcontext
7
+ import math
8
+ from typing import Optional, Tuple
9
+ # from megatron.model import LayerNorm
10
+
11
+ from einops import rearrange
12
+ from easydict import EasyDict as adict
13
+
14
+
15
+ from typing import Optional, Tuple, Type
16
+ from functools import partial
17
+
18
+
19
+
20
+ class MlpProjector(nn.Module):
21
+
22
+ def __init__(self, cfg):
23
+
24
+ super().__init__()
25
+
26
+ self.cfg = cfg
27
+
28
+ if cfg.projector_type == "identity":
29
+ modules = nn.Identity()
30
+
31
+ elif cfg.projector_type == "linear":
32
+ modules = nn.Linear(cfg.input_dim, cfg.n_embed)
33
+
34
+ elif cfg.projector_type == "mlp_gelu":
35
+ mlp_depth = cfg.get("depth", 1)
36
+ modules = [nn.Linear(cfg.input_dim, cfg.n_embed)]
37
+ for _ in range(1, mlp_depth):
38
+ modules.append(nn.GELU())
39
+ modules.append(nn.Linear(cfg.n_embed, cfg.n_embed))
40
+ modules = nn.Sequential(*modules)
41
+
42
+ elif cfg.projector_type == "normlayer_downsample_mlp_gelu":
43
+ mlp_depth = cfg.get("depth", 1)
44
+ mlp_ratio = cfg.get("mlp_ratio", 1)
45
+ modules = [
46
+ nn.LayerNorm(cfg.input_dim * cfg.downsample_ratio * cfg.downsample_ratio),
47
+ nn.Linear(cfg.input_dim * cfg.downsample_ratio * cfg.downsample_ratio, cfg.n_embed * mlp_ratio)
48
+ ]
49
+ for _ in range(1, mlp_depth - 1):
50
+ modules.append(nn.GELU())
51
+ modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed * mlp_ratio))
52
+ modules.append(nn.GELU())
53
+ modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed))
54
+ modules = nn.Sequential(*modules)
55
+
56
+ elif cfg.projector_type == "downsample_mlp_gelu":
57
+ mlp_depth = cfg.get("depth", 1)
58
+ mlp_ratio = cfg.get("mlp_ratio", 1)
59
+ modules = [nn.Linear(cfg.input_dim * cfg.downsample_ratio * cfg.downsample_ratio, cfg.n_embed * mlp_ratio)]
60
+ for _ in range(1, mlp_depth - 1):
61
+ modules.append(nn.GELU())
62
+ modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed * mlp_ratio))
63
+ modules.append(nn.GELU())
64
+ modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed))
65
+ modules = nn.Sequential(*modules)
66
+
67
+ elif cfg.projector_type == "low_high_hybrid_split_mlp_gelu":
68
+ mlp_depth = cfg.get("depth", 1)
69
+ self.high_up_proj = nn.Linear(cfg.input_dim, cfg.n_embed // 2)
70
+ self.low_up_proj = nn.Linear(cfg.input_dim, cfg.n_embed // 2)
71
+
72
+ modules = []
73
+ for _ in range(1, mlp_depth):
74
+ modules.append(nn.GELU())
75
+ modules.append(nn.Linear(cfg.n_embed, cfg.n_embed))
76
+ modules = nn.Sequential(*modules)
77
+
78
+ elif cfg.projector_type == "hybrid_split_feature_mlp_gelu":
79
+ mlp_depth = cfg.get("depth", 1)
80
+ channel_div = cfg.get("channel_div", 0.5)
81
+ self.high_up_proj = nn.Linear(cfg.input_dim[0], int(cfg.n_embed * channel_div))
82
+ self.low_up_proj = nn.Linear(cfg.input_dim[1], cfg.n_embed - int(cfg.n_embed * channel_div))
83
+
84
+ modules = []
85
+ for _ in range(1, mlp_depth):
86
+ modules.append(nn.GELU())
87
+ modules.append(nn.Linear(cfg.n_embed, cfg.n_embed))
88
+ modules = nn.Sequential(*modules)
89
+
90
+ elif cfg.projector_type == "low_high_split_mlp_gelu":
91
+ mlp_depth = cfg.get("depth", 1)
92
+ modules = []
93
+ for _ in range(1, mlp_depth):
94
+ modules.append(nn.GELU())
95
+ modules.append(nn.Linear(cfg.n_embed // 2, cfg.n_embed // 2))
96
+ modules = nn.Sequential(*modules)
97
+ self.high_layers = nn.Sequential(*modules)
98
+ self.low_layers = copy.deepcopy(modules)
99
+
100
+ else:
101
+ raise ValueError(f"Unknown projector type: {cfg.projector_type}")
102
+
103
+ if cfg.get("token_pooling", False):
104
+ self.token_pooling_layer = nn.Linear(cfg.input_dim * 4, cfg.input_dim)
105
+
106
+ if cfg.get("conv_fusion_high_low_features", False):
107
+ self.fusion_layer = nn.Linear(cfg.input_dim, cfg.input_dim)
108
+ self.layers = modules
109
+
110
+ def forward(self, x):
111
+ if self.cfg.get("token_pooling", False):
112
+ batch_size, wxh, channels = x.shape
113
+ w = h = int(wxh**0.5)
114
+ x = x.view(batch_size, w, h, channels)
115
+ x = x.permute(0, 3, 1, 2)
116
+ # import ipdb; ipdb.set_trace()
117
+ patches = x.unfold(2, 2, 2).unfold(3, 2, 2)
118
+ batch_size, channels, h_patches, w_patches, _, _ = patches.size()
119
+ patches = patches.contiguous().view(batch_size, channels, h_patches * w_patches, -1)
120
+
121
+ patches = patches.permute(0, 2, 1, 3).contiguous()
122
+ patches = patches.view(batch_size, h_patches * w_patches, channels * 4)
123
+
124
+ x = self.token_pooling_layer(patches)
125
+
126
+ if self.cfg.get("conv_fusion_high_low_features", False):
127
+ x = self.fusion_layer(x[:, 0]) + x[:, 1]
128
+
129
+ if self.cfg.projector_type == 'low_high_hybrid_split_mlp_gelu':
130
+ high_x, low_x = x[0], x[1]
131
+ high_x = self.high_up_proj(high_x)
132
+ low_x = self.low_up_proj(low_x)
133
+ x = torch.concat([high_x, low_x], dim=-1)
134
+
135
+ if self.cfg.projector_type == 'hybrid_split_feature_mlp_gelu':
136
+ high_x = x[...,:self.cfg.input_dim[0]]
137
+ low_x = x[...,self.cfg.input_dim[0]:]
138
+ high_x = self.high_up_proj(high_x)
139
+ low_x = self.low_up_proj(low_x)
140
+ x = torch.concat([high_x, low_x], dim=-1)
141
+
142
+ if self.cfg.projector_type == 'low_high_split_mlp_gelu':
143
+ high_x, low_x = x[0], x[1]
144
+ high_x = self.high_layers(high_x)
145
+ low_x = self.low_layers(low_x)
146
+ x = torch.concat([high_x, low_x], dim=-1)
147
+ return x
148
+
149
+ if self.cfg.projector_type == 'downsample_mlp_gelu' or self.cfg.projector_type == 'normlayer_downsample_mlp_gelu':
150
+ bs, hw, input_dim = x.shape
151
+ h = w = int((hw) ** 0.5)
152
+
153
+ """compute padding"""
154
+ if h % self.cfg.downsample_ratio:
155
+ pad = self.cfg.downsample_ratio - h % self.cfg.downsample_ratio
156
+ else:
157
+ pad = 0
158
+ x = x.reshape(bs, h, w, input_dim)
159
+ if pad > 0:
160
+ x = F.pad(x, (0, 0, 0, pad, 0, pad), "constant", 0)
161
+
162
+ """4 to 1 concat"""
163
+ x = x.permute(0, 3, 1, 2) # B, C, H, W
164
+ x = F.unfold(x, kernel_size=self.cfg.downsample_ratio, stride=self.cfg.downsample_ratio, padding=0) # B, C*4, HW // 4
165
+ x = x.permute(0, 2, 1)
166
+
167
+ return self.layers(x)
168
+
169
+ @staticmethod
170
+ def get_flops_per_sample(cfg):
171
+ if cfg.projector_type == "linear":
172
+ fwd = 2 * cfg.input_dim * cfg.n_embed
173
+
174
+ elif "mlp_gelu" in cfg.projector_type :
175
+ mlp_depth = cfg.get("depth", 1)
176
+ downsample_ratio = cfg.get("downsample_ratio", 1)
177
+ input_dim = sum(cfg.input_dim) if isinstance(cfg.input_dim, list) else cfg.input_dim
178
+ input_dim = input_dim * downsample_ratio * downsample_ratio
179
+ fwd = 2 * input_dim * cfg.n_embed + (mlp_depth - 1) * 2 * cfg.n_embed * cfg.n_embed
180
+ else:
181
+ fwd = 0
182
+
183
+ return fwd * 3
184
+
185
+
186
+ #===================clip============================================================
187
+
188
+ class LayerNormfp32(torch.nn.LayerNorm):
189
+ """Subclass torch's LayerNorm to handle fp16."""
190
+
191
+ def forward(self, x: torch.Tensor):
192
+ orig_type = x.dtype
193
+ ret = super().forward(x.type(torch.float32))
194
+ return ret.type(orig_type)
195
+
196
+
197
+ def get_abs_pos(abs_pos, tgt_size):
198
+ # abs_pos: L, C
199
+ # tgt_size: M
200
+ # return: M, C
201
+
202
+ # print(tgt_size)
203
+ # print(abs_pos.shape)
204
+ # exit()
205
+ dim = abs_pos.size(-1)
206
+ # print(dim)
207
+ abs_pos_new = abs_pos.squeeze(0)
208
+ cls_token, old_pos_embed = abs_pos_new[:1], abs_pos_new[1:]
209
+
210
+
211
+
212
+ src_size = int(math.sqrt(abs_pos_new.shape[0] - 1))
213
+ tgt_size = int(math.sqrt(tgt_size))
214
+ dtype = abs_pos.dtype
215
+
216
+ if src_size != tgt_size:
217
+ old_pos_embed = old_pos_embed.view(1, src_size, src_size, dim).permute(0, 3, 1,
218
+ 2).contiguous()
219
+ old_pos_embed = old_pos_embed.to(torch.float32)
220
+ new_pos_embed = F.interpolate(
221
+ old_pos_embed,
222
+ size=(tgt_size, tgt_size),
223
+ mode='bicubic',
224
+ antialias=True,
225
+ align_corners=False,
226
+ ).to(dtype)
227
+ new_pos_embed = new_pos_embed.permute(0, 2, 3, 1)
228
+ new_pos_embed = new_pos_embed.view(tgt_size * tgt_size, dim)
229
+ vision_pos_embed = torch.cat([cls_token, new_pos_embed], dim=0)
230
+ vision_pos_embed = vision_pos_embed.view(1, tgt_size * tgt_size + 1, dim)
231
+ return vision_pos_embed
232
+ else:
233
+ return abs_pos
234
+
235
+ @torch.jit.script
236
+ def quick_gelu(x):
237
+ return x * torch.sigmoid(1.702 * x)
238
+
239
+
240
+
241
+ class CLIPVisionEmbeddings(nn.Module):
242
+ def __init__(self, hidden_size=1024, image_size=224, patch_size=14, num_channels=3):
243
+ super().__init__()
244
+ self.embed_dim = hidden_size
245
+ self.image_size = image_size
246
+ self.patch_size = patch_size
247
+
248
+ self.class_embedding = torch.nn.Parameter(torch.randn(self.embed_dim))
249
+
250
+ self.patch_embedding = torch.nn.Conv2d(
251
+ in_channels=num_channels,
252
+ out_channels=self.embed_dim,
253
+ kernel_size=self.patch_size,
254
+ stride=self.patch_size,
255
+ bias=False,
256
+ )
257
+
258
+ self.num_patches = (self.image_size // self.patch_size) ** 2
259
+ self.num_positions = self.num_patches + 1
260
+ self.position_embedding = torch.nn.Embedding(self.num_positions, self.embed_dim)
261
+ self.register_buffer(
262
+ "position_ids", torch.arange(self.num_positions).expand((1, -1))
263
+ )
264
+
265
+ def forward(self, pixel_values, patch_embeds):
266
+ batch_size = pixel_values.shape[0]
267
+ # patch_embeds = self.patch_embedding(
268
+ # pixel_values
269
+ # ) # shape = [*, width, grid, grid]
270
+
271
+
272
+ if patch_embeds is not None:
273
+ patch_embeds = patch_embeds
274
+ # print(patch_embeds.shape)
275
+ else:
276
+ patch_embeds = self.patch_embedding(pixel_values)
277
+ # print(111111)
278
+ # shape = [*, width, grid, grid]
279
+ # patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
280
+
281
+ patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
282
+
283
+
284
+ class_embeds = self.class_embedding.expand(batch_size, 1, -1)
285
+ embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
286
+
287
+ # x = torch.cat([cls_token, x], dim=1)
288
+ embeddings = embeddings + get_abs_pos(self.position_embedding(self.position_ids), embeddings.size(1))
289
+ # embeddings = embeddings + self.position_embedding(self.position_ids)
290
+ return embeddings
291
+
292
+
293
+ class NoTPFeedForward(nn.Module):
294
+ def __init__(
295
+ self,
296
+ cfg,
297
+ dim: int,
298
+ hidden_dim: int,
299
+ ):
300
+ super().__init__()
301
+
302
+ self.fc1 = torch.nn.Linear(dim, hidden_dim, bias=True)
303
+ self.fc2 = torch.nn.Linear(hidden_dim, dim, bias=True)
304
+
305
+ def forward(self, x):
306
+ output = self.fc2(quick_gelu(self.fc1(x)))
307
+ return output
308
+
309
+
310
+
311
+
312
+ class NoTPAttention(torch.nn.Module):
313
+ def __init__(self, cfg):
314
+ super().__init__()
315
+ self.num_heads = cfg.num_attention_heads
316
+ self.n_local_heads = cfg.num_attention_heads
317
+ self.head_dim = cfg.hidden_size // cfg.num_attention_heads
318
+ self.max_seq_len = cfg.seq_length
319
+ self.use_flash_attention = cfg.use_flash_attn
320
+
321
+ self.qkv_proj = torch.nn.Linear(cfg.hidden_size, cfg.hidden_size * 3, bias=True)
322
+ self.out_proj = torch.nn.Linear(cfg.hidden_size, cfg.hidden_size, bias=True)
323
+
324
+ # self.core_attention = CoreAttention(cfg, AttnType.self_attn)
325
+
326
+ self.attn_drop = cfg.attention_dropout
327
+
328
+ def forward(
329
+ self,
330
+ x: torch.Tensor,
331
+ ):
332
+ bsz, seqlen, _ = x.shape
333
+ xqkv = self.qkv_proj(x)
334
+ xqkv = xqkv.view(bsz, seqlen, 3, self.num_heads, self.head_dim)
335
+
336
+ if self.use_flash_attention:
337
+
338
+ xq, xk, xv = torch.split(xqkv, 1, dim=2)
339
+ xq = xq.squeeze(2)
340
+ xk = xk.squeeze(2)
341
+ xv = xv.squeeze(2)
342
+ # xq, xk, xv = xqkv[:, :, 0, ...], xqkv[:, :, 1, ...], xqkv[:, :, 2, ...]
343
+
344
+ # (B, num_head, S, head_size)
345
+ xq = xq.permute(0, 2, 1, 3)
346
+ xk = xk.permute(0, 2, 1, 3)
347
+ xv = xv.permute(0, 2, 1, 3)
348
+ # with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):
349
+ output = torch.nn.functional.scaled_dot_product_attention(xq, xk, xv, attn_mask=None)
350
+ output = output.permute(0, 2, 1, 3).reshape(bsz, seqlen, -1)
351
+ # output = output.permute(0, 2, 1, 3).contiguous().view(bsz, seqlen, -1)
352
+ else:
353
+ # print(22222)
354
+ xq, xk, xv = torch.split(xqkv, 1, dim=2)
355
+ xq = xq.squeeze(2)
356
+ xk = xk.squeeze(2)
357
+ xv = xv.squeeze(2)
358
+ # xq, xk, xv = xqkv[:, :, 0, ...], xqkv[:, :, 1, ...], xqkv[:, :, 2, ...]
359
+
360
+ # (B, num_head, S, head_size)
361
+ xq = xq.permute(0, 2, 1, 3)
362
+ xk = xk.permute(0, 2, 1, 3)
363
+ xv = xv.permute(0, 2, 1, 3)
364
+ # with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):
365
+ output = torch.nn.functional.scaled_dot_product_attention(xq, xk, xv, attn_mask=None)
366
+ output = output.permute(0, 2, 1, 3).reshape(bsz, seqlen, -1)
367
+ # output = output.permute(0, 2, 1, 3).contiguous().view(bsz, seqlen, -1)
368
+ output = self.out_proj(output)
369
+ return output
370
+
371
+ class NoTPTransformerBlock(nn.Module):
372
+ def __init__(self, cfg, layer_id: int, multiple_of=256):
373
+ super().__init__()
374
+
375
+ self.n_heads = cfg.num_attention_heads
376
+ self.dim = cfg.hidden_size
377
+ self.head_dim = cfg.hidden_size // cfg.num_attention_heads
378
+ self.self_attn = NoTPAttention(cfg)
379
+ self.mlp = NoTPFeedForward(
380
+ cfg, dim=cfg.hidden_size, hidden_dim=cfg.ffn_hidden_size
381
+ )
382
+ self.layer_id = layer_id
383
+ self.layer_norm1 = torch.nn.LayerNorm(
384
+ cfg.hidden_size, eps=cfg.layernorm_epsilon
385
+ )
386
+ self.layer_norm2 = torch.nn.LayerNorm(
387
+ cfg.hidden_size, eps=cfg.layernorm_epsilon
388
+ )
389
+
390
+ def forward(self, x: torch.Tensor):
391
+ residual = self.self_attn.forward(self.layer_norm1(x))
392
+ h = x + residual
393
+ out = h + self.mlp.forward(self.layer_norm2(h))
394
+ return out
395
+
396
+
397
+ class NoTPTransformer(nn.Module):
398
+ def __init__(self, cfg):
399
+ super().__init__()
400
+
401
+ self.cfg = cfg
402
+ # self.recompute_list = self.cfg.get("recompute_list", [])
403
+ self.num_layers = cfg.num_layers # _get_num_layers(cfg)
404
+
405
+ self.layers = torch.nn.ModuleList()
406
+ for layer_id in range(self.num_layers):
407
+ self.layers.append(
408
+ NoTPTransformerBlock(
409
+ cfg,
410
+ layer_id + 1,
411
+ )
412
+ )
413
+
414
+ def forward(
415
+ self,
416
+ hidden_states,
417
+ ):
418
+
419
+ for lid, layer in enumerate(self.layers):
420
+ # if lid in self.recompute_list:
421
+ # def custom(layer_id):
422
+ # def custom_forward(*args, **kwargs):
423
+ # x_ = self.layers[layer_id](*args, **kwargs)
424
+ # return x_
425
+
426
+ # return custom_forward
427
+
428
+ # assert hidden_states.requires_grad == True, logger.warning(
429
+ # "When using recalculation, the input must have grad fn"
430
+ # )
431
+ # hidden_states = tensor_parallel.checkpoint(
432
+ # custom(lid),
433
+ # False,
434
+ # hidden_states.contiguous()
435
+ # )
436
+ # else:
437
+ hidden_states = layer(hidden_states)
438
+
439
+ return hidden_states
440
+
441
+
442
+ # from megatron.core.tensor_parallel.layers import non_tensor_paralleled, local_dp_reduce, local_dp_scatter
443
+
444
+ class VitModel(nn.Module):
445
+ def __init__(
446
+ self,
447
+ cfg,
448
+ freeze_embed=False,
449
+ freeze_pre_norm=False
450
+ ) -> None:
451
+ super().__init__()
452
+
453
+ self.embeddings = CLIPVisionEmbeddings(hidden_size=cfg.hidden_size, image_size=cfg.image_size, patch_size=cfg.patch_size)
454
+
455
+ if freeze_embed:
456
+ for name, param in self.embeddings.named_parameters():
457
+ param.requires_grad = False
458
+
459
+ self.transformer = NoTPTransformer(cfg=cfg)
460
+
461
+ if cfg.get("fp32norm", False):
462
+ logger.info("Load fp32 layernorm for ViT.")
463
+ self.pre_layrnorm = LayerNormfp32(
464
+ cfg.hidden_size,
465
+ eps=cfg.get("pre_layernorm_epsilon", 1e-5),
466
+ )
467
+ else:
468
+ self.pre_layrnorm = torch.nn.LayerNorm(
469
+ cfg.hidden_size,
470
+ eps=cfg.get("pre_layernorm_epsilon", 1e-5),
471
+ )
472
+
473
+ # self.pre_layrnorm = RMSNorm(
474
+ # cfg.hidden_size,
475
+ # eps=cfg.get("pre_layernorm_epsilon", 1e-5),
476
+ # sequence_parallel=False,
477
+ # use_fp32=True,
478
+ # use_optimus=True,
479
+ # )
480
+
481
+ if freeze_pre_norm:
482
+ for name, param in self.pre_layrnorm.named_parameters():
483
+ param.requires_grad = False
484
+
485
+ for p in self.parameters():
486
+ p.micro_dp = True
487
+
488
+ def set_input_tensor(self, input_tensor):
489
+ if not isinstance(input_tensor, list):
490
+ input_tensor = [input_tensor]
491
+ self.transformer.set_input_tensor(input_tensor[0])
492
+
493
+ def __str__(self) -> str:
494
+ return "open_clip"
495
+
496
+ def forward(
497
+ self,
498
+ x,
499
+ patch_embeds
500
+ ):
501
+ x = self.embeddings(x, patch_embeds)
502
+ hidden_states = self.pre_layrnorm(x)
503
+
504
+ # hidden_states, dis = local_dp_scatter(hidden_states)
505
+ output = self.transformer(hidden_states)
506
+
507
+ # output = local_dp_reduce(output, dis)
508
+
509
+ return output
510
+
511
+
512
+ vit_model_cfg = adict(
513
+ num_layers=24,
514
+ hidden_size=1024,
515
+ num_heads = 16,
516
+ num_attention_heads=16,
517
+ ffn_hidden_size=4096,
518
+ seq_length=256,
519
+ max_position_embeddings=256,
520
+ use_flash_attn=False,
521
+ understand_projector_stride=2,
522
+ hidden_dropout = 0.0,
523
+ attention_dropout = 0.0,
524
+ no_persist_layer_norm = False,
525
+ layernorm_epsilon = 1e-5,
526
+ pre_layernorm_epsilon = 1e-5,
527
+ image_size = 224,
528
+ patch_size = 14,
529
+ recompute_list = []
530
+ )
531
+
532
+ def build_clip_l(config=None):
533
+ cfg = copy.deepcopy(vit_model_cfg)
534
+ if config is not None:
535
+ spec = config.vision_config['width']['clip-l-14-224']
536
+ cfg.update(num_layers=spec['layers'], hidden_size=spec['width'],
537
+ num_heads=spec['heads'], num_attention_heads=spec['heads'],
538
+ ffn_hidden_size=spec['width'] * 4)
539
+ return VitModel(
540
+ cfg=cfg,
541
+ freeze_embed=False,
542
+ freeze_pre_norm=False,
543
+ )
544
+
545
+
546
+
547
+
548
+
549
+ #=========================Sam-Vary=================================
550
+
551
+
552
+ def get_abs_pos_sam(abs_pos, tgt_size):
553
+
554
+ dtype = abs_pos.dtype
555
+
556
+ src_size = abs_pos.size(1)
557
+
558
+ if src_size != tgt_size:
559
+ old_pos_embed = abs_pos.permute(0, 3, 1, 2)
560
+ old_pos_embed = old_pos_embed.to(torch.float32)
561
+ new_pos_embed = F.interpolate(
562
+ old_pos_embed,
563
+ size=(tgt_size, tgt_size),
564
+ mode='bicubic',
565
+ antialias=True,
566
+ align_corners=False,
567
+ ).to(dtype)
568
+ new_pos_embed = new_pos_embed.permute(0, 2, 3, 1)
569
+ return new_pos_embed
570
+ else:
571
+ return abs_pos
572
+
573
+
574
+
575
+
576
+ class MLPBlock(nn.Module):
577
+ def __init__(
578
+ self,
579
+ embedding_dim: int,
580
+ mlp_dim: int,
581
+ act: Type[nn.Module] = nn.GELU,
582
+ ) -> None:
583
+ super().__init__()
584
+ self.lin1 = nn.Linear(embedding_dim, mlp_dim)
585
+ self.lin2 = nn.Linear(mlp_dim, embedding_dim)
586
+ self.act = act()
587
+
588
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
589
+ return self.lin2(self.act(self.lin1(x)))
590
+
591
+
592
+ # From https://github.com/facebookresearch/detectron2/blob/main/detectron2/layers/batch_norm.py # noqa
593
+ # Itself from https://github.com/facebookresearch/ConvNeXt/blob/d1fa8f6fef0a165b27399986cc2bdacc92777e40/models/convnext.py#L119 # noqa
594
+ class LayerNorm2d(nn.Module):
595
+ def __init__(self, num_channels: int, eps: float = 1e-6) -> None:
596
+ super().__init__()
597
+ self.weight = nn.Parameter(torch.ones(num_channels))
598
+ self.bias = nn.Parameter(torch.zeros(num_channels))
599
+ self.eps = eps
600
+
601
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
602
+ u = x.mean(1, keepdim=True)
603
+ s = (x - u).pow(2).mean(1, keepdim=True)
604
+ x = (x - u) / torch.sqrt(s + self.eps)
605
+ x = self.weight[:, None, None] * x + self.bias[:, None, None]
606
+ return x
607
+
608
+
609
+ # This class and its supporting functions below lightly adapted from the ViTDet backbone available at: https://github.com/facebookresearch/detectron2/blob/main/detectron2/modeling/backbone/vit.py # noqa
610
+ class ImageEncoderViT(nn.Module):
611
+ def __init__(
612
+ self,
613
+ img_size: int = 1024,
614
+ patch_size: int = 16,
615
+ in_chans: int = 3,
616
+ embed_dim: int = 768,
617
+ depth: int = 12,
618
+ num_heads: int = 12,
619
+ mlp_ratio: float = 4.0,
620
+ out_chans: int = 256,
621
+ qkv_bias: bool = True,
622
+ norm_layer: Type[nn.Module] = nn.LayerNorm,
623
+ act_layer: Type[nn.Module] = nn.GELU,
624
+ use_abs_pos: bool = True,
625
+ use_rel_pos: bool = False,
626
+ rel_pos_zero_init: bool = True,
627
+ window_size: int = 0,
628
+ global_attn_indexes: Tuple[int, ...] = (),
629
+ ) -> None:
630
+ """
631
+ Args:
632
+ img_size (int): Input image size.
633
+ patch_size (int): Patch size.
634
+ in_chans (int): Number of input image channels.
635
+ embed_dim (int): Patch embedding dimension.
636
+ depth (int): Depth of ViT.
637
+ num_heads (int): Number of attention heads in each ViT block.
638
+ mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
639
+ qkv_bias (bool): If True, add a learnable bias to query, key, value.
640
+ norm_layer (nn.Module): Normalization layer.
641
+ act_layer (nn.Module): Activation layer.
642
+ use_abs_pos (bool): If True, use absolute positional embeddings.
643
+ use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
644
+ rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
645
+ window_size (int): Window size for window attention blocks.
646
+ global_attn_indexes (list): Indexes for blocks using global attention.
647
+ """
648
+ super().__init__()
649
+ self.img_size = img_size
650
+
651
+ self.patch_embed = PatchEmbed(
652
+ kernel_size=(patch_size, patch_size),
653
+ stride=(patch_size, patch_size),
654
+ in_chans=in_chans,
655
+ embed_dim=embed_dim,
656
+ )
657
+
658
+ self.pos_embed: Optional[nn.Parameter] = None
659
+ if use_abs_pos:
660
+ # Initialize absolute positional embedding with pretrain image size.
661
+ self.pos_embed = nn.Parameter(
662
+ torch.zeros(1, img_size // patch_size, img_size // patch_size, embed_dim)
663
+ )
664
+
665
+ self.blocks = nn.ModuleList()
666
+ for i in range(depth):
667
+ block = Block(
668
+ dim=embed_dim,
669
+ num_heads=num_heads,
670
+ mlp_ratio=mlp_ratio,
671
+ qkv_bias=qkv_bias,
672
+ norm_layer=norm_layer,
673
+ act_layer=act_layer,
674
+ use_rel_pos=use_rel_pos,
675
+ rel_pos_zero_init=rel_pos_zero_init,
676
+ window_size=window_size if i not in global_attn_indexes else 0,
677
+ input_size=(img_size // patch_size, img_size // patch_size),
678
+ )
679
+ self.blocks.append(block)
680
+
681
+ self.neck = nn.Sequential(
682
+ nn.Conv2d(
683
+ embed_dim,
684
+ out_chans,
685
+ kernel_size=1,
686
+ bias=False,
687
+ ),
688
+ LayerNorm2d(out_chans),
689
+ nn.Conv2d(
690
+ out_chans,
691
+ out_chans,
692
+ kernel_size=3,
693
+ padding=1,
694
+ bias=False,
695
+ ),
696
+ LayerNorm2d(out_chans),
697
+ )
698
+
699
+ self.net_2 = nn.Conv2d(256, 512, kernel_size=3, stride=2, padding=1, bias=False)
700
+ self.net_3 = nn.Conv2d(512, 1024, kernel_size=3, stride=2, padding=1, bias=False)
701
+
702
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
703
+ x = self.patch_embed(x)
704
+ if self.pos_embed is not None:
705
+ # x = x + self.pos_embed
706
+ x = x + get_abs_pos_sam(self.pos_embed, x.size(1))
707
+
708
+ for blk in self.blocks:
709
+ x = blk(x)
710
+
711
+ x = self.neck(x.permute(0, 3, 1, 2))
712
+ x2 = self.net_2(x)
713
+ x3 = self.net_3(x2.clone())
714
+
715
+ return x3
716
+
717
+
718
+ class Block(nn.Module):
719
+ """Transformer blocks with support of window attention and residual propagation blocks"""
720
+
721
+ def __init__(
722
+ self,
723
+ dim: int,
724
+ num_heads: int,
725
+ mlp_ratio: float = 4.0,
726
+ qkv_bias: bool = True,
727
+ norm_layer: Type[nn.Module] = nn.LayerNorm,
728
+ act_layer: Type[nn.Module] = nn.GELU,
729
+ use_rel_pos: bool = False,
730
+ rel_pos_zero_init: bool = True,
731
+ window_size: int = 0,
732
+ input_size: Optional[Tuple[int, int]] = None,
733
+ ) -> None:
734
+ """
735
+ Args:
736
+ dim (int): Number of input channels.
737
+ num_heads (int): Number of attention heads in each ViT block.
738
+ mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
739
+ qkv_bias (bool): If True, add a learnable bias to query, key, value.
740
+ norm_layer (nn.Module): Normalization layer.
741
+ act_layer (nn.Module): Activation layer.
742
+ use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
743
+ rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
744
+ window_size (int): Window size for window attention blocks. If it equals 0, then
745
+ use global attention.
746
+ input_size (tuple(int, int) or None): Input resolution for calculating the relative
747
+ positional parameter size.
748
+ """
749
+ super().__init__()
750
+ self.norm1 = norm_layer(dim)
751
+ self.attn = Attention(
752
+ dim,
753
+ num_heads=num_heads,
754
+ qkv_bias=qkv_bias,
755
+ use_rel_pos=use_rel_pos,
756
+ rel_pos_zero_init=rel_pos_zero_init,
757
+ input_size=input_size if window_size == 0 else (window_size, window_size),
758
+ )
759
+
760
+ self.norm2 = norm_layer(dim)
761
+ self.mlp = MLPBlock(embedding_dim=dim, mlp_dim=int(dim * mlp_ratio), act=act_layer)
762
+
763
+ self.window_size = window_size
764
+
765
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
766
+ shortcut = x
767
+ x = self.norm1(x)
768
+ # Window partition
769
+ if self.window_size > 0:
770
+ H, W = x.shape[1], x.shape[2]
771
+ x, pad_hw = window_partition(x, self.window_size)
772
+
773
+ x = self.attn(x)
774
+ # Reverse window partition
775
+ if self.window_size > 0:
776
+ x = window_unpartition(x, self.window_size, pad_hw, (H, W))
777
+
778
+ x = shortcut + x
779
+ x = x + self.mlp(self.norm2(x))
780
+
781
+ return x
782
+
783
+
784
+ class Attention(nn.Module):
785
+ """Multi-head Attention block with relative position embeddings."""
786
+
787
+ def __init__(
788
+ self,
789
+ dim: int,
790
+ num_heads: int = 8,
791
+ qkv_bias: bool = True,
792
+ use_rel_pos: bool = False,
793
+ rel_pos_zero_init: bool = True,
794
+ input_size: Optional[Tuple[int, int]] = None,
795
+ ) -> None:
796
+ """
797
+ Args:
798
+ dim (int): Number of input channels.
799
+ num_heads (int): Number of attention heads.
800
+ qkv_bias (bool): If True, add a learnable bias to query, key, value.
801
+ rel_pos (bool): If True, add relative positional embeddings to the attention map.
802
+ rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
803
+ input_size (tuple(int, int) or None): Input resolution for calculating the relative
804
+ positional parameter size.
805
+ """
806
+ super().__init__()
807
+ self.num_heads = num_heads
808
+ head_dim = dim // num_heads
809
+ self.scale = head_dim**-0.5
810
+
811
+ self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
812
+ self.proj = nn.Linear(dim, dim)
813
+
814
+ self.use_rel_pos = use_rel_pos
815
+ if self.use_rel_pos:
816
+ assert (
817
+ input_size is not None
818
+ ), "Input size must be provided if using relative positional encoding."
819
+ # initialize relative positional embeddings
820
+ self.rel_pos_h = nn.Parameter(torch.zeros(2 * input_size[0] - 1, head_dim))
821
+ self.rel_pos_w = nn.Parameter(torch.zeros(2 * input_size[1] - 1, head_dim))
822
+
823
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
824
+ B, H, W, _ = x.shape
825
+ # qkv with shape (3, B, nHead, H * W, C)
826
+ qkv = self.qkv(x).reshape(B, H * W, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)
827
+ # q, k, v with shape (B * nHead, H * W, C)
828
+ q, k, v = qkv.reshape(3, B * self.num_heads, H * W, -1).unbind(0)
829
+
830
+ rel_h, rel_w = None, None
831
+ if self.use_rel_pos:
832
+ rel_h, rel_w = add_decomposed_rel_pos(q, self.rel_pos_h, self.rel_pos_w, (H, W), (H, W))
833
+
834
+ q = q.view(B, self.num_heads, H * W, -1)
835
+ k = k.view(B, self.num_heads, H * W, -1)
836
+ v = v.view(B, self.num_heads, H * W, -1)
837
+
838
+ if self.use_rel_pos:
839
+ rel_h = rel_h.view(B, self.num_heads, rel_h.size(1), rel_h.size(2), rel_h.size(3))
840
+ rel_w = rel_w.view(B, self.num_heads, rel_w.size(1), rel_w.size(2), rel_w.size(3))
841
+ attn_bias = (rel_h + rel_w).view(B, self.num_heads, rel_h.size(2), rel_h.size(3) * rel_w.size(4))
842
+ x = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=attn_bias)
843
+ # x = _attention_rel_h_rel_w(q, k, v, rel_h, rel_w)
844
+ else:
845
+ x = torch.nn.functional.scaled_dot_product_attention(q, k, v)
846
+
847
+ x = x.view(B, self.num_heads, H, W, -1).permute(0, 2, 3, 1, 4).reshape(B, H, W, -1)
848
+
849
+ x = self.proj(x)
850
+
851
+ return x
852
+
853
+
854
+ def window_partition(x: torch.Tensor, window_size: int) -> Tuple[torch.Tensor, Tuple[int, int]]:
855
+ """
856
+ Partition into non-overlapping windows with padding if needed.
857
+ Args:
858
+ x (tensor): input tokens with [B, H, W, C].
859
+ window_size (int): window size.
860
+
861
+ Returns:
862
+ windows: windows after partition with [B * num_windows, window_size, window_size, C].
863
+ (Hp, Wp): padded height and width before partition
864
+ """
865
+ B, H, W, C = x.shape
866
+
867
+ pad_h = (window_size - H % window_size) % window_size
868
+ pad_w = (window_size - W % window_size) % window_size
869
+ if pad_h > 0 or pad_w > 0:
870
+ x = F.pad(x, (0, 0, 0, pad_w, 0, pad_h))
871
+ Hp, Wp = H + pad_h, W + pad_w
872
+
873
+ x = x.view(B, Hp // window_size, window_size, Wp // window_size, window_size, C)
874
+ windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)
875
+ return windows, (Hp, Wp)
876
+
877
+
878
+ def window_unpartition(
879
+ windows: torch.Tensor, window_size: int, pad_hw: Tuple[int, int], hw: Tuple[int, int]
880
+ ) -> torch.Tensor:
881
+ """
882
+ Window unpartition into original sequences and removing padding.
883
+ Args:
884
+ windows (tensor): input tokens with [B * num_windows, window_size, window_size, C].
885
+ window_size (int): window size.
886
+ pad_hw (Tuple): padded height and width (Hp, Wp).
887
+ hw (Tuple): original height and width (H, W) before padding.
888
+
889
+ Returns:
890
+ x: unpartitioned sequences with [B, H, W, C].
891
+ """
892
+ Hp, Wp = pad_hw
893
+ H, W = hw
894
+ B = windows.shape[0] // (Hp * Wp // window_size // window_size)
895
+ x = windows.view(B, Hp // window_size, Wp // window_size, window_size, window_size, -1)
896
+ x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, Hp, Wp, -1)
897
+
898
+ if Hp > H or Wp > W:
899
+ x = x[:, :H, :W, :].contiguous()
900
+ return x
901
+
902
+
903
+ def get_rel_pos(q_size: int, k_size: int, rel_pos: torch.Tensor) -> torch.Tensor:
904
+ """
905
+ Get relative positional embeddings according to the relative positions of
906
+ query and key sizes.
907
+ Args:
908
+ q_size (int): size of query q.
909
+ k_size (int): size of key k.
910
+ rel_pos (Tensor): relative position embeddings (L, C).
911
+
912
+ Returns:
913
+ Extracted positional embeddings according to relative positions.
914
+ """
915
+ max_rel_dist = int(2 * max(q_size, k_size) - 1)
916
+ # Interpolate rel pos if needed.
917
+ if rel_pos.shape[0] != max_rel_dist:
918
+ # Interpolate rel pos.
919
+ dtype = rel_pos.dtype
920
+ rel_pos = rel_pos.to(torch.float32)
921
+ rel_pos_resized = F.interpolate(
922
+ rel_pos.reshape(1, rel_pos.shape[0], -1).permute(0, 2, 1),
923
+ size=max_rel_dist,
924
+ mode="linear",
925
+ ).to(dtype)
926
+ rel_pos_resized = rel_pos_resized.reshape(-1, max_rel_dist).permute(1, 0)
927
+ else:
928
+ rel_pos_resized = rel_pos
929
+
930
+ # Scale the coords with short length if shapes for q and k are different.
931
+ q_coords = torch.arange(q_size, device=rel_pos.device)[:, None] * max(k_size / q_size, 1.0)
932
+ k_coords = torch.arange(k_size, device=rel_pos.device)[None, :] * max(q_size / k_size, 1.0)
933
+ relative_coords = (q_coords - k_coords) + (k_size - 1) * max(q_size / k_size, 1.0)
934
+
935
+ return rel_pos_resized[relative_coords.long()]
936
+
937
+
938
+ def add_decomposed_rel_pos(
939
+ q: torch.Tensor,
940
+ rel_pos_h: torch.Tensor,
941
+ rel_pos_w: torch.Tensor,
942
+ q_size: Tuple[int, int],
943
+ k_size: Tuple[int, int],
944
+ ) -> torch.Tensor:
945
+ """
946
+ Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`.
947
+ https://github.com/facebookresearch/mvit/blob/19786631e330df9f3622e5402b4a419a263a2c80/mvit/models/attention.py # noqa B950
948
+ Args:
949
+ q (Tensor): query q in the attention layer with shape (B, q_h * q_w, C).
950
+ rel_pos_h (Tensor): relative position embeddings (Lh, C) for height axis.
951
+ rel_pos_w (Tensor): relative position embeddings (Lw, C) for width axis.
952
+ q_size (Tuple): spatial sequence size of query q with (q_h, q_w).
953
+ k_size (Tuple): spatial sequence size of key k with (k_h, k_w).
954
+
955
+ Returns:
956
+ attn (Tensor): attention map with added relative positional embeddings.
957
+ """
958
+ q_h, q_w = q_size
959
+ k_h, k_w = k_size
960
+ Rh = get_rel_pos(q_h, k_h, rel_pos_h)
961
+ Rw = get_rel_pos(q_w, k_w, rel_pos_w)
962
+
963
+ B, _, dim = q.shape
964
+ r_q = q.reshape(B, q_h, q_w, dim)
965
+ rel_h = torch.einsum("bhwc,hkc->bhwk", r_q, Rh)
966
+ rel_w = torch.einsum("bhwc,wkc->bhwk", r_q, Rw)
967
+ rel_h = rel_h.unsqueeze(-1)
968
+ rel_w = rel_w.unsqueeze(-2)
969
+ rel_h = rel_h.reshape(B, q_h * q_w, k_h, 1)
970
+ rel_w = rel_w.reshape(B, q_h * q_w, 1, k_w)
971
+
972
+ return rel_h, rel_w
973
+
974
+
975
+ class PatchEmbed(nn.Module):
976
+ """
977
+ Image to Patch Embedding.
978
+ """
979
+
980
+ def __init__(
981
+ self,
982
+ kernel_size: Tuple[int, int] = (16, 16),
983
+ stride: Tuple[int, int] = (16, 16),
984
+ padding: Tuple[int, int] = (0, 0),
985
+ in_chans: int = 3,
986
+ embed_dim: int = 768,
987
+ ) -> None:
988
+ """
989
+ Args:
990
+ kernel_size (Tuple): kernel size of the projection layer.
991
+ stride (Tuple): stride of the projection layer.
992
+ padding (Tuple): padding size of the projection layer.
993
+ in_chans (int): Number of input image channels.
994
+ embed_dim (int): Patch embedding dimension.
995
+ """
996
+ super().__init__()
997
+
998
+ self.proj = nn.Conv2d(
999
+ in_chans, embed_dim, kernel_size=kernel_size, stride=stride, padding=padding
1000
+ )
1001
+
1002
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
1003
+ x = self.proj(x)
1004
+ # B C H W -> B H W C
1005
+ x = x.permute(0, 2, 3, 1)
1006
+ return x
1007
+
1008
+
1009
+ def build_sam_vit_b(checkpoint=None, config=None):
1010
+ if config is not None:
1011
+ spec = config.vision_config['width']['sam_vit_b']
1012
+ encoder = ImageEncoderViT(embed_dim=spec['width'], depth=spec['layers'],
1013
+ num_heads=spec['heads'], global_attn_indexes=spec['global_attn_indexes'],
1014
+ out_chans=spec['neck_channels'], window_size=14, use_rel_pos=True,
1015
+ norm_layer=partial(torch.nn.LayerNorm, eps=1e-6))
1016
+ c1, c2 = spec['downsample_channels']
1017
+ encoder.net_2 = nn.Conv2d(spec['neck_channels'], c1, 3, stride=2, padding=1, bias=False)
1018
+ encoder.net_3 = nn.Conv2d(c1, c2, 3, stride=2, padding=1, bias=False)
1019
+ return encoder
1020
+ return _build_sam(
1021
+ encoder_embed_dim=768,
1022
+ encoder_depth=12,
1023
+ encoder_num_heads=12,
1024
+ encoder_global_attn_indexes=[2, 5, 8, 11],
1025
+ checkpoint=checkpoint,
1026
+ )
1027
+
1028
+ def build_sam_fast_vit_b(checkpoint=None, compile_mode='max-autotune', dtype=torch.bfloat16):
1029
+ image_encoder = build_sam_vit_b(checkpoint).eval().to(dtype)
1030
+ # sam = _apply_eval_dtype_sam(sam, dtype)
1031
+ image_encoder = torch.compile(image_encoder, mode=compile_mode)
1032
+ return image_encoder
1033
+
1034
+
1035
+ def _build_sam(
1036
+ encoder_embed_dim,
1037
+ encoder_depth,
1038
+ encoder_num_heads,
1039
+ encoder_global_attn_indexes,
1040
+ checkpoint=None,
1041
+ ):
1042
+ prompt_embed_dim = 256
1043
+ image_size = 1024
1044
+ vit_patch_size = 16
1045
+ image_embedding_size = image_size // vit_patch_size
1046
+ image_encoder=ImageEncoderViT(
1047
+ depth=encoder_depth,
1048
+ embed_dim=encoder_embed_dim,
1049
+ img_size=image_size,
1050
+ mlp_ratio=4,
1051
+ norm_layer=partial(torch.nn.LayerNorm, eps=1e-6),
1052
+ num_heads=encoder_num_heads,
1053
+ patch_size=vit_patch_size,
1054
+ qkv_bias=True,
1055
+ use_rel_pos=True,
1056
+ global_attn_indexes=encoder_global_attn_indexes,
1057
+ window_size=14,
1058
+ out_chans=prompt_embed_dim,
1059
+ )
1060
+ image_encoder.eval()
1061
+ if checkpoint is not None:
1062
+ # with open(checkpoint, "rb") as f:
1063
+ state_dict = torch.load(checkpoint)
1064
+ # print(state_dict.keys())
1065
+ # for key in state_dict:
1066
+ # image_encoder.load_state_dict({k[14:]: v for k, v in state_dict.items() if 'image_encoder' in k}, strict=False)
1067
+ # ocr-anyting
1068
+ # image_encoder.load_state_dict(state_dict, strict=True)
1069
+ # tob
1070
+ image_encoder.load_state_dict({k[30:]: v for k, v in state_dict.items() if 'vision_tower_high' in k}, strict=True)
1071
+ print(checkpoint)
1072
+ return image_encoder
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 0,
4
+ "eos_token_id": 1,
5
+ "transformers_version": "4.52.1"
6
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:35106c601d5693dc748814318feefbf609ea8a15b00e3ade28dd8296551ebd7f
3
+ size 18123048
modeling_deepseekv2.py ADDED
@@ -0,0 +1,531 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """DASH overlay on top of the page-isolated (page2d) base modeling.
2
+
3
+ Thin monkey-patch: `_sparse_base_impl.py` carries the full base implementation; this file
4
+ only swaps the **prefill** path of SlidingWindowLlamaAttention for
5
+ dash_attention.DashAttention (entmax soft prior + log(prior) bias, gradients reaching the
6
+ router straight from the LM loss, no KL teacher) when
7
+ `sparse_attention_config.training_stage == "dash"`. Decode, the non-dash stages and all
8
+ other classes keep using the base implementation.
9
+
10
+ Three details the dash path relies on -- page isolation degrades silently if any of them
11
+ is dropped:
12
+
13
+ 1. The main-branch rope goes through `self._main_rope(...)` rather than
14
+ `self.rotary_emb(...)`, otherwise sparse_main_rope_mode=page2d is bypassed and the main
15
+ branch falls back to 1D.
16
+ 2. `self.indexer` is **not** set to None: `_main_rope_coords` returns None without an
17
+ indexer (main branch falls back to 1D), and every page/rope helper
18
+ (_build_page_aware_ids / _build_page_rope_pos / rotary_emb_2d /
19
+ _build_kl_candidate_mask) lives on the indexer.
20
+ 3. Router scoring uses the indexer's own 2-axis rope (sparse_rope_*: one page pair at
21
+ theta=256 with the page index taken mod 2, plus 63 token pairs at theta=1e4, resampled)
22
+ together with a page-aware PER-QUERY candidate domain [B,S,nblk].
23
+ """
24
+
25
+ import copy
26
+ import math
27
+ import os
28
+
29
+ import torch
30
+ import torch.nn.functional as F
31
+
32
+ if os.environ.get("DASH_DEBUG_ANOMALY") == "1":
33
+ torch.autograd.set_detect_anomaly(True)
34
+ print("[DASH] torch.autograd anomaly detection ENABLED", flush=True)
35
+
36
+ # Re-export the ENTIRE base namespace (including underscore helpers) so that
37
+ # modeling_unlimitedocr.py's `from .modeling_deepseekv2 import <name>` keeps working.
38
+ from . import _sparse_base_impl as _base # noqa: E402
39
+
40
+ globals().update({k: v for k, v in vars(_base).items() if not k.startswith("__")})
41
+
42
+ from ._sparse_base_impl import ( # noqa: E402
43
+ DeepseekV2PreTrainedModel as _BasePreTrained,
44
+ SlidingWindowLlamaAttention as _BaseAttn,
45
+ _llama_apply_rotary_pos_emb,
46
+ _llama_repeat_kv,
47
+ )
48
+ from .dash_attention import DashAttention # noqa: E402
49
+ from .dash_attention import entmax15 as _entmax15 # noqa: E402
50
+ from .dash_attention import _SCORE_NEG, _ACT_CLAMP, _MASK_NEG # noqa: E402
51
+
52
+ _orig_init = _BaseAttn.__init__
53
+ _orig_forward = _BaseAttn.forward
54
+ _DASH_LOGGED = False
55
+ _IDX_ROPE_WARNED = False
56
+ # id(sparse_attention_config) -> deep copy taken BEFORE DashAttention.__init__ rewrites it
57
+ # (_persist_router_cfg writes the live process state -- k_start=64, env DASH_STAGE0 default
58
+ # "indexer" -- over dash_topk_blocks / dash_stage0 / ...). Inference reads the snapshot.
59
+ _CKPT_SP = {}
60
+ # Inference switch, toggled by UnlimitedOCRForCausalLM.infer (see configure_dash_inference).
61
+ # When on, DASH layers route exactly like vLLM's `unlimited_page2d_dash` plugin
62
+ # (dash_router.DashRouterMath.route_and_select: top-M entmax window + hard top-K read set)
63
+ # in prefill AND decode; decode no longer falls back to dense full-cache attention.
64
+ _INFER = {"on": False}
65
+ _ENTMAX_NEG = -1.0e4
66
+
67
+
68
+ def _patched_init(self, config, layer_idx):
69
+ sp = getattr(config, "sparse_attention_config", None)
70
+ is_dash = isinstance(sp, dict) and sp.get("training_stage") == "dash"
71
+ if is_dash and id(sp) not in _CKPT_SP:
72
+ _CKPT_SP[id(sp)] = copy.deepcopy(sp)
73
+ if is_dash:
74
+ _saved_stage = sp.get("training_stage")
75
+ sp["training_stage"] = "warmup"
76
+ _orig_init(self, config, layer_idx)
77
+ if is_dash:
78
+ sp["training_stage"] = _saved_stage
79
+
80
+ self.dash_router = None
81
+ if is_dash and self.indexer is not None:
82
+ self.dash_router = DashAttention(config)
83
+ global _DASH_LOGGED
84
+ if not _DASH_LOGGED:
85
+ _DASH_LOGGED = True
86
+ print(f"[DASH] differentiable DASH attention active "
87
+ f"(entmax router, block_size={self.dash_router.block_size}, "
88
+ f"local_window={self.dash_router.local_window}, "
89
+ f"index_dim={self.dash_router.index_dim}, "
90
+ f"main_rope={self.main_rope_mode}, "
91
+ f"idx_rope={self.indexer.rope_mode}, "
92
+ f"page_bucket={self.indexer.image_bucket_constraint}); "
93
+ f"first dash layer_idx={layer_idx}", flush=True)
94
+
95
+
96
+ def _dash_page_ids(self, idx, src_ids, key_length):
97
+ """Page-aware ids, built with the same calls as the base prefill path."""
98
+ if idx.image_bucket_constraint:
99
+ itm, ipi, qpi = idx._build_page_aware_ids(
100
+ src_ids, idx.image_token_id, idx.page_token_id, key_length)
101
+ else:
102
+ itm = idx._build_flat_image_token_mask(src_ids, idx.image_token_id, key_length)
103
+ ipi = qpi = None
104
+ idx._log_image_mask(itm, src_ids, key_length)
105
+ return itm, ipi, qpi
106
+
107
+
108
+ def _dash_block_cand(self, idx, router, pos_ids, itm, ipi, qpi, nblk, bsz, q_len):
109
+ """PER-QUERY candidate domain [B,S,nblk], reusing the indexer's
110
+ _build_kl_candidate_mask (same-page image bucket + per-query fallback). Built in
111
+ q_chunk slices so the [B,S,nblk,block_size] intermediate is never materialised at
112
+ once (32k x 1024 x 32 bool is roughly 1 GiB)."""
113
+ if itm is None and (ipi is None or qpi is None):
114
+ return None
115
+ step = max(1, int(router.q_chunk))
116
+ out = []
117
+ for c0 in range(0, q_len, step):
118
+ c1 = min(c0 + step, q_len)
119
+ cm = idx._build_kl_candidate_mask(
120
+ pos_ids[:, c0:c1], nblk, itm, bsz, c1 - c0,
121
+ image_token_page_ids=ipi, query_token_page_ids=qpi) # [B,1,C,nblk]
122
+ out.append(cm[:, 0])
123
+ return torch.cat(out, dim=1) # [B,S,nblk]
124
+
125
+
126
+ def _dash_indexer_rope(self, idx, router, x, pos_ids, coords, main_pe):
127
+ """cos/sin used for router scoring: the indexer's 2-axis rope (sparse_rope_*) when
128
+ the coordinates are available, otherwise the indexer's 1D rope. With
129
+ index_dim < head_dim the two axes cannot be sliced (cat(ang, ang) + rotate_half would
130
+ mix the phases of the two axes), so 1D is used instead."""
131
+ global _IDX_ROPE_WARNED
132
+ if coords is not None and getattr(idx, "rotary_emb_2d", None) is not None:
133
+ if router.index_dim == idx.head_dim:
134
+ return idx.rotary_emb_2d(x, coords[0], coords[1])
135
+ if not _IDX_ROPE_WARNED:
136
+ _IDX_ROPE_WARNED = True
137
+ print(f"[PageRope][WARN] sparse_index_dim={router.index_dim} != "
138
+ f"head_dim={idx.head_dim}: the 2-axis cos/sin cannot be sliced per "
139
+ f"frequency pair, router scoring falls back to 1D RoPE "
140
+ f"(the main branch is unaffected)", flush=True)
141
+ if getattr(idx, "rotary_emb_1d", None) is not None:
142
+ return idx.rotary_emb_1d(x, pos_ids)
143
+ return main_pe
144
+
145
+
146
+ def _dash_attn_forward(self, hidden_states, attention_mask, position_ids, past_kv):
147
+ idx = self.indexer
148
+ router = self.dash_router
149
+ num_heads = self.config.num_attention_heads
150
+ num_kv_heads = self.config.num_key_value_heads
151
+ head_dim = self.head_dim
152
+ num_kv_groups = self.num_key_value_groups
153
+ bsz, q_len, _ = hidden_states.size()
154
+
155
+ pos_ids = position_ids
156
+ if _base._pk.is_packed(pos_ids):
157
+ raise RuntimeError(
158
+ "[DASH] packing rows are not supported by the dash path "
159
+ "(fused e2e kernel requires attn_mask=None, and page rope would run the "
160
+ "<PAGE> cumsum across sample boundaries). Disable packing.")
161
+
162
+ q = self.q_proj(hidden_states).view(bsz, q_len, num_heads, head_dim).transpose(1, 2)
163
+ key_states = self.k_proj(hidden_states).view(bsz, q_len, num_kv_heads, head_dim).transpose(1, 2)
164
+ value_states = self.v_proj(hidden_states).view(bsz, q_len, num_kv_heads, head_dim).transpose(1, 2)
165
+
166
+ src_ids = getattr(self, '_sparse_source_input_ids', None)
167
+ itm, ipi, qpi = _dash_page_ids(self, idx, src_ids, q_len)
168
+
169
+ coords = None
170
+ if idx.rope_mode == "page2d":
171
+ coords = self._main_rope_pos_2d
172
+ if coords is not None and coords[0].shape[-1] != q_len:
173
+ coords = None
174
+ if coords is None:
175
+ coords = idx._build_page_rope_pos(src_ids, q_len)
176
+ idx._rope_pos_2d = coords
177
+ if self.main_rope_mode == "page2d" and coords is not None:
178
+ self._main_rope_pos_2d = coords
179
+ cos, sin = self._main_rope(value_states, pos_ids, past_kv, q_len)
180
+ else:
181
+ self._main_rope_pos_2d = None
182
+ cos, sin = self.rotary_emb(value_states, pos_ids)
183
+ q, key_states = _llama_apply_rotary_pos_emb(q, key_states, cos, sin)
184
+
185
+ if past_kv is not None:
186
+ key_states, value_states = past_kv.update(key_states, value_states, self.layer_idx)
187
+ k = _llama_repeat_kv(key_states, num_kv_groups)
188
+ v = _llama_repeat_kv(value_states, num_kv_groups)
189
+ idx_pe = _dash_indexer_rope(self, idx, router, value_states, pos_ids, coords, (cos, sin))
190
+
191
+ nblk = -(-k.shape[2] // router.block_size)
192
+ block_cand = _dash_block_cand(self, idx, router, pos_ids, itm, ipi, qpi,
193
+ nblk, bsz, q_len)
194
+
195
+ router_mask = attention_mask
196
+ if router_mask is not None and bool((router_mask[:, :, -1, :] > -1.0).all()):
197
+ router_mask = None
198
+
199
+ if _INFER["on"]:
200
+ attn_output = _infer_prefill_attend(self, router, hidden_states, q, k, v, idx_pe,
201
+ pos_ids, block_cand, router_mask, past_kv)
202
+ else:
203
+ attn_output = router(hidden_states, q, k, v,
204
+ position_embeddings=idx_pe,
205
+ position_ids=pos_ids,
206
+ attention_mask=router_mask,
207
+ image_block_mask=None,
208
+ block_cand=block_cand,
209
+ page_ids=(ipi, qpi))
210
+ attn_output = attn_output.transpose(1, 2).contiguous().reshape(bsz, q_len, -1)
211
+ attn_output = self.o_proj(attn_output)
212
+
213
+ return attn_output, None, past_kv, None
214
+
215
+
216
+ def _patched_forward(self, *args, **kwargs):
217
+ if self.dash_router is not None:
218
+ def _get(name, idx_, default=None):
219
+ if name in kwargs:
220
+ return kwargs[name]
221
+ if len(args) > idx_:
222
+ return args[idx_]
223
+ return default
224
+
225
+ hidden_states = _get('hidden_states', 0)
226
+ attention_mask = _get('attention_mask', 1)
227
+ position_ids = _get('position_ids', 2)
228
+ past_kv = _get('past_key_value', 3)
229
+ if past_kv is None:
230
+ past_kv = kwargs.get('past_key_values', None)
231
+
232
+ bsz, q_len, _ = hidden_states.size()
233
+ if q_len > 1:
234
+ if position_ids is None:
235
+ position_ids = torch.arange(
236
+ q_len, device=hidden_states.device).unsqueeze(0).expand(bsz, -1)
237
+ return _dash_attn_forward(self, hidden_states, attention_mask,
238
+ position_ids, past_kv)
239
+ if (_INFER["on"] and past_kv is not None
240
+ and self.layer_idx in getattr(past_kv, "_dash_idx_k", {})):
241
+ return _dash_decode_forward(self, hidden_states, position_ids, past_kv)
242
+
243
+ return _orig_forward(self, *args, **kwargs)
244
+
245
+
246
+ _BaseAttn.__init__ = _patched_init
247
+ _BaseAttn.forward = _patched_forward
248
+
249
+ # ---------------------------------------------------------------------------
250
+ # ---------------------------------------------------------------------------
251
+ _orig_init_weights = _BasePreTrained._init_weights
252
+
253
+
254
+ def _patched_init_weights(self, module):
255
+ _orig_init_weights(self, module)
256
+ if isinstance(module, DashAttention):
257
+ module.head_cls.data.zero_()
258
+
259
+
260
+ _BasePreTrained._init_weights = _patched_init_weights
261
+
262
+
263
+ # ===========================================================================
264
+ # Inference path -- parity with vLLM's `unlimited_page2d_dash` plugin
265
+ # (plugin_page2d_dash/unlimited_page2d_dash: dash_router.py + dash_attention_layer.py).
266
+ # Only active between configure_dash_inference(model, True) / (model, False), i.e. inside
267
+ # UnlimitedOCRForCausalLM.infer; training never enters these functions.
268
+ #
269
+ # vs. the training forward (DashAttention.forward) the inference router differs in:
270
+ # * routing params come from the checkpoint snapshot (_CKPT_SP): dash_topk_blocks (hard
271
+ # top-K read budget), dash_stage0, per-layer dash_scale_per_layer / dash_floor_per_layer
272
+ # -- the live config dict has been overwritten by _persist_router_cfg at construction;
273
+ # * entmax runs on the top-M (DASH_INFER_TOPM, deploy default 16) scores, then the read
274
+ # set is the first K candidate slots (DashRouterMath.route_and_select, budget=topk);
275
+ # * decode (q_len == 1) is ALSO block-sparse (vLLM DASH_DECODE_DENSE=0): head_cls block
276
+ # summaries over the cached router keys, page-aware candidate mask, bias-aware attend
277
+ # over the selected blocks + the token-level local window.
278
+ # ===========================================================================
279
+
280
+
281
+ def _sched_slot(sp, layer_idx):
282
+ freq = sp.get("sparse_attention_freq")
283
+ if not isinstance(freq, (list, tuple)) or layer_idx >= len(freq):
284
+ return int(layer_idx)
285
+ return sum(1 for f in freq[:layer_idx] if f)
286
+
287
+
288
+ def _per_layer_val(sp, key, layer_idx):
289
+ arr = sp.get(key)
290
+ if not isinstance(arr, (list, tuple)) or not arr:
291
+ return None
292
+ slot = _sched_slot(sp, int(layer_idx))
293
+ return float(arr[slot]) if 0 <= slot < len(arr) else None
294
+
295
+
296
+ def configure_dash_inference(decoder, on, topm=16, page_scope_prefill=True, state=None):
297
+ """Switch the vLLM-parity DASH inference path on/off for ``decoder`` (DeepseekV2Model).
298
+
299
+ on=True returns a state dict; pass it back with on=False to restore.
300
+ """
301
+ if not on:
302
+ if state is not None:
303
+ _base._PAGE_SCOPE_PREFILL = state["page_scope_prefill"]
304
+ for attn, stage0 in state["stage0"]:
305
+ attn.dash_router.stage0 = stage0
306
+ _INFER["on"] = False
307
+ return None
308
+ state = {"page_scope_prefill": _base._PAGE_SCOPE_PREFILL, "stage0": []}
309
+ cfg = decoder.config
310
+ live = getattr(cfg, "sparse_attention_config", None) or {}
311
+ sp = _CKPT_SP.get(id(live), live)
312
+ stage0 = str(os.environ.get("DASH_STAGE0") or sp.get("dash_stage0") or "indexer").lower()
313
+ budget = sp.get("dash_topk_blocks") or sp.get("dash_image_fallback_remote") or 12
314
+ for layer in decoder.layers:
315
+ attn = getattr(layer, "self_attn", None)
316
+ router = getattr(attn, "dash_router", None)
317
+ if router is None:
318
+ continue
319
+ li = attn.layer_idx
320
+ scale = _per_layer_val(sp, "dash_scale_per_layer", li)
321
+ floor = _per_layer_val(sp, "dash_floor_per_layer", li)
322
+ if scale is None:
323
+ scale = float(sp.get("dash_scaling_factor", router.scaling_factor))
324
+ if floor is None:
325
+ floor = float(sp.get("dash_prior_floor", router._prior_floor))
326
+ router._infer_params = dict(
327
+ idx_scale=float(scale) / math.sqrt(router.index_dim), floor=float(floor),
328
+ sigma=float(sp.get("dash_sigma", router.sigma)), K=int(budget), topm=int(topm))
329
+ state["stage0"].append((attn, router.stage0))
330
+ router.stage0 = stage0
331
+ # vLLM page_state.py isolates PREFILL queries too (image query -> its own page, prompt
332
+ # marker -> raw page, trailing instruction -> unconstrained) == training PAGE_SCOPE_PREFILL=1.
333
+ _base._PAGE_SCOPE_PREFILL = bool(page_scope_prefill)
334
+ _INFER["on"] = True
335
+ return state
336
+
337
+
338
+ @torch.no_grad()
339
+ def _infer_route_select(block_scores, block_cand, p):
340
+ """vLLM DashRouterMath.route_and_select (budget_topk=True) with training's candidacy.
341
+
342
+ block_scores [B,Lq,nblk] fp32 (_SCORE_NEG = not a remote block); block_cand
343
+ [B,Lq,nblk] bool page-aware candidate domain or None.
344
+ Returns (topi [B,Lq,K] int64, bias [B,Lq,K] fp32, valid [B,Lq,K] bool).
345
+ """
346
+ remote = block_scores > (_SCORE_NEG * 0.5)
347
+ cand = remote if block_cand is None else (remote & block_cand)
348
+ cand = torch.where(~cand.any(dim=-1, keepdim=True), remote, cand)
349
+ x = torch.where(cand, block_scores * p["idx_scale"], block_scores.new_full((), _ENTMAX_NEG))
350
+ x = torch.nan_to_num(x, nan=_ENTMAX_NEG, posinf=_ENTMAX_NEG, neginf=_ENTMAX_NEG)
351
+ nblk = x.shape[-1]
352
+ M = max(1, min(int(p["topm"]), nblk)) if int(p["topm"]) > 0 else nblk
353
+ xv, xi = x.topk(M, dim=-1) # descending
354
+ prior = _entmax15(xv, dim=-1) # tau* over the top-M window
355
+ is_cand = cand.gather(-1, xi)
356
+ K = max(1, min(int(p["K"]), M))
357
+ xi, prior, valid = xi[..., :K], prior[..., :K], is_cand[..., :K]
358
+ logp = torch.log(prior.clamp_min(p["floor"]))
359
+ cnt = valid.sum(dim=-1, keepdim=True).clamp(min=1)
360
+ mu = torch.where(valid, logp, torch.zeros_like(logp)).sum(dim=-1, keepdim=True) / cnt
361
+ bias = torch.where(valid, (logp - mu) * (1.0 / p["sigma"]), torch.zeros_like(logp))
362
+ return xi, bias.float(), valid
363
+
364
+
365
+ def _router_qk(router, hidden_states, pe):
366
+ """Router index q/k exactly as DashAttention._block_scores_headcls builds them."""
367
+ B, S, _ = hidden_states.shape
368
+ d = router.index_dim
369
+ hs = torch.nan_to_num(hidden_states, nan=0.0, posinf=0.0, neginf=0.0)
370
+ idx_q = router.q_norm(router.q_proj(hs).view(B, S, 1, d)).transpose(1, 2)
371
+ idx_k = router.k_norm(router.k_proj(hs).view(B, S, 1, d)).transpose(1, 2)
372
+ if pe is not None:
373
+ cos, sin = pe
374
+ idx_q, idx_k = _llama_apply_rotary_pos_emb(idx_q, idx_k, cos[..., :d], sin[..., :d])
375
+ idx_q = torch.nan_to_num(idx_q[:, 0].float(), nan=0.0, posinf=0.0, neginf=0.0)
376
+ idx_k = torch.nan_to_num(idx_k[:, 0].float(), nan=0.0, posinf=0.0, neginf=0.0)
377
+ return idx_q, idx_k
378
+
379
+
380
+ @torch.no_grad()
381
+ def _block_scores_infer(router, idx_q, idx_k, q_pos, nblk):
382
+ """[B,Lq,nblk] fp32 remote block scores for queries ``idx_q`` at positions ``q_pos``
383
+ against ALL keys ``idx_k`` [B,S,d] (prefill: S == Lq; decode: the cached keys).
384
+ Same math as DashAttention._block_scores_headcls / _block_scores."""
385
+ B, S, d = idx_k.shape
386
+ bs = router.block_size
387
+ inv_sqrt_d = 1.0 / math.sqrt(d)
388
+ Sp = nblk * bs
389
+ blk_ids = torch.arange(nblk, device=idx_k.device)
390
+ out = []
391
+ if router.stage0 == "head_cls":
392
+ if Sp > S:
393
+ kk = F.pad(idx_k, (0, 0, 0, Sp - S))
394
+ real = F.pad(idx_k.new_ones(B, S, dtype=torch.bool), (0, Sp - S), value=False)
395
+ else:
396
+ kk = idx_k[:, :Sp]
397
+ real = idx_k.new_ones(B, Sp, dtype=torch.bool)
398
+ kk = kk.view(B, nblk, bs, d)
399
+ vmask = real.view(B, nblk, bs)
400
+ hc = router.head_cls.to(kk.dtype).view(1, 1, 1, d)
401
+ aw = ((kk * hc).sum(-1) * inv_sqrt_d).clamp(-_ACT_CLAMP, _ACT_CLAMP)
402
+ neg = torch.finfo(aw.dtype).min
403
+ row_max = torch.where(vmask, aw, aw.new_full((), neg)).amax(dim=-1, keepdim=True)
404
+ row_max = torch.where(vmask.any(dim=-1, keepdim=True), row_max, torch.zeros_like(row_max))
405
+ e = torch.exp(aw - row_max) * vmask.to(aw.dtype)
406
+ w = e / e.sum(dim=-1, keepdim=True).clamp_min(1e-6)
407
+ k_bar = torch.nan_to_num((w.unsqueeze(-1) * kk).sum(dim=2), nan=0.0, posinf=0.0, neginf=0.0)
408
+ chunk_valid = vmask.any(dim=-1) # [B,nblk]
409
+ for c0 in range(0, idx_q.shape[1], router.q_chunk):
410
+ c1 = min(c0 + router.q_chunk, idx_q.shape[1])
411
+ z = torch.matmul(idx_q[:, c0:c1], k_bar.transpose(-1, -2)) * inv_sqrt_d
412
+ z = z.clamp(-_ACT_CLAMP, _ACT_CLAMP)
413
+ qblk = q_pos[:, c0:c1] // bs
414
+ keep = (blk_ids[None, None, :] <= (qblk[:, :, None] - router.local_blocks)) \
415
+ & chunk_valid[:, None, :]
416
+ out.append(z.masked_fill(~keep, _SCORE_NEG))
417
+ return torch.cat(out, dim=1)
418
+ # stage0 == "indexer": block-max of token-level scores, remote-only (causal minus local)
419
+ kpos = torch.arange(S, device=idx_k.device)
420
+ for c0 in range(0, idx_q.shape[1], router.q_chunk):
421
+ c1 = min(c0 + router.q_chunk, idx_q.shape[1])
422
+ sc = torch.matmul(idx_q[:, c0:c1], idx_k.transpose(-1, -2)).clamp(-_ACT_CLAMP, _ACT_CLAMP)
423
+ pos = q_pos[:, c0:c1]
424
+ future = kpos[None, None, :] > pos[:, :, None]
425
+ local = (kpos[None, None, :] > (pos[:, :, None] - router.local_window)) & ~future
426
+ sc = sc.masked_fill(future | local, _SCORE_NEG)
427
+ if Sp > S:
428
+ sc = F.pad(sc, (0, Sp - S), value=_SCORE_NEG)
429
+ out.append(sc.view(B, c1 - c0, nblk, bs).amax(dim=-1))
430
+ return torch.cat(out, dim=1)
431
+
432
+
433
+ def _infer_prefill_attend(self, router, hidden_states, q, k, v, idx_pe, pos_ids,
434
+ block_cand, attn_mask, past_kv):
435
+ """Prefill (q_len > 1) under _INFER: vLLM-parity routing, training attend kernels."""
436
+ B, H, S, D = q.shape
437
+ nblk = (S + router.block_size - 1) // router.block_size
438
+ p = router._infer_params
439
+ with torch.autocast(device_type=q.device.type, enabled=False):
440
+ idx_q, idx_k = _router_qk(router, hidden_states, idx_pe)
441
+ if past_kv is not None:
442
+ if not hasattr(past_kv, "_dash_idx_k"):
443
+ past_kv._dash_idx_k = {}
444
+ past_kv._dash_idx_k[self.layer_idx] = idx_k
445
+ sc = _block_scores_infer(router, idx_q, idx_k, pos_ids, nblk)
446
+ topi, bias, valid = _infer_route_select(sc, block_cand, p)
447
+ if router.use_triton and q.is_cuda and attn_mask is None:
448
+ out = router._forward_triton(q, k, v, topi, bias, valid, S, nblk, pos_ids)
449
+ if out is not None:
450
+ return out
451
+ return router._forward_dense(q, k, v, topi, bias, valid, S, nblk, attn_mask)
452
+
453
+
454
+ @torch.no_grad()
455
+ def _decode_attend(q, k_all, v_all, topi, bias, valid, q_pos, block_size, local_window, scale):
456
+ """Bias-aware block-sparse attention for decode queries (fp32 math).
457
+
458
+ Selected remote blocks add their ``bias``; the token-level local window
459
+ (q_pos - local_window, q_pos] is attended without bias and wins on overlap --
460
+ same read set / weights as vLLM dash_sparse_attn decode and training _forward_dense.
461
+ """
462
+ B, H, Lq, D = q.shape
463
+ S = k_all.shape[2]
464
+ nblk = (S + block_size - 1) // block_size
465
+ kpos = torch.arange(S, device=q.device)
466
+ src = torch.where(valid, bias, bias.new_full((), _MASK_NEG))
467
+ blk = torch.full((B, Lq, nblk), _MASK_NEG, device=q.device, dtype=torch.float32)
468
+ blk = blk.scatter(-1, topi.long(), src.float())
469
+ base = blk.index_select(-1, kpos // block_size) # [B,Lq,S]
470
+ qp = q_pos[:, :, None]
471
+ causal = kpos[None, None, :] <= qp
472
+ local = causal & (kpos[None, None, :] > (qp - local_window))
473
+ keep = causal & (local | (base > _MASK_NEG * 0.5))
474
+ m = torch.where(local, torch.zeros_like(base), base).masked_fill(~keep, float("-inf"))
475
+ kk = k_all.float()
476
+ vv = v_all.float()
477
+ if kk.shape[1] != H:
478
+ kk = _llama_repeat_kv(kk, H // kk.shape[1])
479
+ vv = _llama_repeat_kv(vv, H // vv.shape[1])
480
+ s = torch.matmul(q.float(), kk.transpose(-1, -2)) * scale + m[:, None]
481
+ return torch.matmul(torch.softmax(s, dim=-1), vv) # [B,H,Lq,D]
482
+
483
+
484
+ def _dash_decode_forward(self, hidden_states, position_ids, past_kv):
485
+ """Decode step (q_len == 1) of a DASH layer under _INFER (vLLM DASH_DECODE_DENSE=0)."""
486
+ idx = self.indexer
487
+ router = self.dash_router
488
+ nh = self.config.num_attention_heads
489
+ nkv = self.config.num_key_value_heads
490
+ hd = self.head_dim
491
+ bsz, q_len, _ = hidden_states.size()
492
+ if position_ids is None:
493
+ past = past_kv.get_seq_length(self.layer_idx)
494
+ position_ids = torch.arange(past, past + q_len, device=hidden_states.device)[None].expand(bsz, -1)
495
+
496
+ q = self.q_proj(hidden_states).view(bsz, q_len, nh, hd).transpose(1, 2)
497
+ k = self.k_proj(hidden_states).view(bsz, q_len, nkv, hd).transpose(1, 2)
498
+ v = self.v_proj(hidden_states).view(bsz, q_len, nkv, hd).transpose(1, 2)
499
+
500
+ # (tok_pos, page_pos) from the incremental decode state machine (advances it once).
501
+ coords = None
502
+ if self.main_rope_mode == "page2d" and self.rotary_emb_2d_main is not None:
503
+ coords = self._main_rope_coords(q_len, past_kv)
504
+ if coords is not None:
505
+ cos, sin = self.rotary_emb_2d_main(v, coords[0], coords[1])
506
+ else:
507
+ cos, sin = self.rotary_emb(v, position_ids)
508
+ q, k = _llama_apply_rotary_pos_emb(q, k, cos, sin)
509
+ k_all, v_all = past_kv.update(k, v, self.layer_idx)
510
+ idx_pe = _dash_indexer_rope(self, idx, router, v, position_ids,
511
+ coords if idx.rope_mode == "page2d" else None, (cos, sin))
512
+
513
+ p = router._infer_params
514
+ with torch.autocast(device_type=hidden_states.device.type, enabled=False):
515
+ idx_q, idx_k_new = _router_qk(router, hidden_states, idx_pe)
516
+ idx_k = torch.cat([past_kv._dash_idx_k[self.layer_idx], idx_k_new], dim=1)
517
+ past_kv._dash_idx_k[self.layer_idx] = idx_k
518
+ S = k_all.shape[2]
519
+ nblk = (S + router.block_size - 1) // router.block_size
520
+ sc = _block_scores_infer(router, idx_q, idx_k, position_ids, nblk) # [B,Lq,nblk]
521
+ allow = getattr(self, "_dash_dec_allow", None) # [B,nblk] or None
522
+ cand = None
523
+ if allow is not None and router.image_only:
524
+ if allow.shape[-1] < nblk:
525
+ allow = F.pad(allow, (0, nblk - allow.shape[-1]), value=False)
526
+ cand = allow[:, None, :nblk].expand(bsz, q_len, nblk)
527
+ topi, bias, valid = _infer_route_select(sc, cand, p)
528
+ out = _decode_attend(q, k_all, v_all, topi, bias, valid, position_ids,
529
+ router.block_size, router.local_window, router._scale)
530
+ out = out.to(hidden_states.dtype).transpose(1, 2).contiguous().reshape(bsz, q_len, -1)
531
+ return self.o_proj(out), None, past_kv, None
modeling_unlimitedocr.py ADDED
@@ -0,0 +1,1228 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from .modeling_deepseekv2 import (
2
+ DeepseekV2Model, DeepseekV2ForCausalLM, _fused_chunked_lm_ce,
3
+ _td, _td_now, _td_add, _td_report,
4
+ )
5
+ from .modeling_deepseekv2 import _INFER as _DASH_INFER, configure_dash_inference
6
+ from .configuration_deepseek_v2 import DeepseekV2Config
7
+ # transformers' dynamic-module loader only copies DIRECT relative imports
8
+ # of the entry module to the HF cache for local model dirs (no recursion). Force
9
+ # sparse_gather_triton (used lazily inside modeling_deepseekv2) to be copied by
10
+ # importing it here at the entry point. The regex in transformers only matches
11
+ # ``from .xxx import yyy`` (not ``from . import xxx``), so keep this exact form.
12
+ try:
13
+ from .sparse_gather_triton import SPARSE_BLOCK_SIZE_SENTINEL as _sgt_sentinel # noqa: F401
14
+ except Exception:
15
+ pass
16
+ # same reason — packing_patch is used inside modeling_deepseekv2
17
+ # (2nd-level import), so force it to be copied by importing it here at the entry.
18
+ try:
19
+ from .packing_patch import is_packed as _pk_copy_hint # noqa: F401
20
+ except Exception:
21
+ pass
22
+ # Same reason for the dash overlay: _sparse_base_impl (the base implementation),
23
+ # dash_attention (entmax router) and sparse_gather_e2e_triton (fused kernel with the
24
+ # per-block bias) are 2nd-level relative imports as well, so they are only copied
25
+ # into the HF cache when the entry module imports them directly.
26
+ try:
27
+ from ._sparse_base_impl import DeepseekV2Model as _dash_base_copy_hint # noqa: F401
28
+ except Exception:
29
+ pass
30
+ try:
31
+ from .dash_attention import DashAttention as _dash_router_copy_hint # noqa: F401
32
+ except Exception:
33
+ pass
34
+ try:
35
+ from .sparse_gather_e2e_triton import triton_gather_attention_e2e as _dash_e2e_copy_hint # noqa: F401
36
+ except Exception:
37
+ pass
38
+ from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
39
+ from typing import List, Optional, Tuple, Union
40
+ from dataclasses import dataclass
41
+
42
+ from transformers.cache_utils import Cache
43
+ import requests
44
+ from PIL import Image, ImageOps, ImageDraw, ImageFont
45
+ from io import BytesIO
46
+ import torch
47
+ import torch.nn as nn
48
+ import torch.nn.functional as F
49
+ from torch.nn import CrossEntropyLoss
50
+ from torchvision import transforms
51
+ from torchvision.transforms.functional import InterpolationMode
52
+ import os
53
+ from .deepencoder import build_sam_vit_b, build_clip_l, MlpProjector
54
+ from addict import Dict
55
+ from transformers import TextStreamer
56
+ from .conversation import get_conv_template
57
+ from abc import ABC
58
+ import math
59
+ import re
60
+
61
+
62
+ @dataclass
63
+ class CausalLMOutputWithPastAndKL(CausalLMOutputWithPast):
64
+ """ForCausalLM output with an extra `indexer_kl_loss` field so the sparse
65
+ indexer's KL alignment loss survives DDP's pytree flatten/unflatten (which strips
66
+ any dynamic attributes attached to a plain `CausalLMOutputWithPast`).
67
+ Read by `swift/trainers/seq2seq_trainer.py:compute_loss`.
68
+ """
69
+ indexer_kl_loss: Optional[torch.FloatTensor] = None
70
+ # raw fused-CE token-accuracy counts (correct / valid). Kept as separate
71
+ # counts (not a pre-averaged ratio) so the trainer can token-weight the accuracy
72
+ # exactly like swift's native compute_acc path.
73
+ token_acc_correct: Optional[torch.FloatTensor] = None
74
+ token_acc_valid: Optional[torch.FloatTensor] = None
75
+
76
+ from tqdm import tqdm
77
+ import numpy as np
78
+ import time
79
+
80
+
81
+ def load_image(image_path):
82
+
83
+ try:
84
+ image = Image.open(image_path)
85
+
86
+ corrected_image = ImageOps.exif_transpose(image)
87
+
88
+ return corrected_image
89
+
90
+ except Exception as e:
91
+ print(f"error: {e}")
92
+ try:
93
+ return Image.open(image_path)
94
+ except:
95
+ return None
96
+
97
+
98
+ def re_match(text):
99
+ ref_pattern = r'(<\|ref\|>(.*?)<\|/ref\|><\|det\|>(.*?)<\|/det\|>)'
100
+ matches = re.findall(ref_pattern, text, re.DOTALL)
101
+
102
+ det_pattern = r'(<\|det\|>\s*([A-Za-z_][\w-]*)\s*(\[[^\]]+\])\s*<\|/det\|>)'
103
+ for full_match, label, box in re.findall(det_pattern, text, re.DOTALL):
104
+ matches.append((full_match, label, box))
105
+
106
+ mathes_image = []
107
+ mathes_other = []
108
+ for a_match in matches:
109
+ if a_match[1].strip() == 'image' or '<|ref|>image<|/ref|>' in a_match[0]:
110
+ mathes_image.append(a_match[0])
111
+ else:
112
+ mathes_other.append(a_match[0])
113
+ return matches, mathes_image, mathes_other
114
+
115
+
116
+ def extract_coordinates_and_label(ref_text, image_width, image_height):
117
+
118
+ try:
119
+ label_type = ref_text[1]
120
+ cor_list = eval(ref_text[2])
121
+ if cor_list and isinstance(cor_list[0], (int, float)):
122
+ cor_list = [cor_list]
123
+ except Exception as e:
124
+ print(e)
125
+ return None
126
+
127
+ return (label_type, cor_list)
128
+
129
+
130
+ def draw_bounding_boxes(image, refs, ouput_path, image_prefix=''):
131
+
132
+ image_width, image_height = image.size
133
+
134
+ img_draw = image.copy()
135
+ draw = ImageDraw.Draw(img_draw)
136
+
137
+ overlay = Image.new('RGBA', img_draw.size, (0, 0, 0, 0))
138
+ draw2 = ImageDraw.Draw(overlay)
139
+
140
+ # try:
141
+ # except IOError:
142
+ # try:
143
+ # font = ImageFont.truetype("DejaVuSans.ttf", 20)
144
+ # except IOError:
145
+ font = ImageFont.load_default()
146
+
147
+ img_idx = 0
148
+
149
+ for i, ref in enumerate(refs):
150
+ try:
151
+ result = extract_coordinates_and_label(ref, image_width, image_height)
152
+ if result:
153
+ label_type, points_list = result
154
+
155
+ color = (np.random.randint(0, 200), np.random.randint(0, 200), np.random.randint(0, 255))
156
+
157
+ color_a = color + (20, )
158
+ for points in points_list:
159
+ x1, y1, x2, y2 = points
160
+
161
+ x1 = int(x1 / 999 * image_width)
162
+ y1 = int(y1 / 999 * image_height)
163
+
164
+ x2 = int(x2 / 999 * image_width)
165
+ y2 = int(y2 / 999 * image_height)
166
+
167
+ if label_type == 'image':
168
+ try:
169
+ cropped = image.crop((x1, y1, x2, y2))
170
+ cropped.save(f"{ouput_path}/images/{image_prefix}{img_idx}.jpg")
171
+ except Exception as e:
172
+ print(e)
173
+ pass
174
+ img_idx += 1
175
+
176
+ try:
177
+ if label_type == 'title':
178
+ draw.rectangle([x1, y1, x2, y2], outline=color, width=4)
179
+ draw2.rectangle([x1, y1, x2, y2], fill=color_a, outline=(0, 0, 0, 0), width=1)
180
+ else:
181
+ draw.rectangle([x1, y1, x2, y2], outline=color, width=2)
182
+ draw2.rectangle([x1, y1, x2, y2], fill=color_a, outline=(0, 0, 0, 0), width=1)
183
+ text_x = x1
184
+ text_y = max(0, y1 - 15)
185
+
186
+
187
+ text_bbox = draw.textbbox((0, 0), label_type, font=font)
188
+ text_width = text_bbox[2] - text_bbox[0]
189
+ text_height = text_bbox[3] - text_bbox[1]
190
+ draw.rectangle([text_x, text_y, text_x + text_width, text_y + text_height],
191
+ fill=(255, 255, 255, 30))
192
+
193
+ draw.text((text_x, text_y), label_type, font=font, fill=color)
194
+ except:
195
+ pass
196
+ except:
197
+ continue
198
+ img_draw.paste(overlay, (0, 0), overlay)
199
+ return img_draw
200
+
201
+
202
+ def process_image_with_refs(image, ref_texts, output_path, image_prefix=''):
203
+
204
+ result_image = draw_bounding_boxes(image, ref_texts, output_path, image_prefix=image_prefix)
205
+
206
+ return result_image
207
+
208
+
209
+
210
+
211
+
212
+ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
213
+ best_ratio_diff = float('inf')
214
+ best_ratio = (1, 1)
215
+ area = width * height
216
+ for ratio in target_ratios:
217
+ target_aspect_ratio = ratio[0] / ratio[1]
218
+ ratio_diff = abs(aspect_ratio - target_aspect_ratio)
219
+ if ratio_diff < best_ratio_diff:
220
+ best_ratio_diff = ratio_diff
221
+ best_ratio = ratio
222
+ elif ratio_diff == best_ratio_diff:
223
+ if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
224
+ best_ratio = ratio
225
+ # print(f'width: {width}, height: {height}, best_ratio: {best_ratio}')
226
+ return best_ratio
227
+
228
+
229
+ def dynamic_preprocess(image, min_num=2, max_num=16, image_size=640, use_thumbnail=False):
230
+ orig_width, orig_height = image.size
231
+ aspect_ratio = orig_width / orig_height
232
+
233
+ # calculate the existing image aspect ratio
234
+ target_ratios = set(
235
+ (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
236
+ i * j <= max_num and i * j >= min_num)
237
+ # print(target_ratios)
238
+ target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
239
+
240
+ # find the closest aspect ratio to the target
241
+ target_aspect_ratio = find_closest_aspect_ratio(
242
+ aspect_ratio, target_ratios, orig_width, orig_height, image_size)
243
+
244
+ # print(target_aspect_ratio)
245
+ # calculate the target width and height
246
+ target_width = image_size * target_aspect_ratio[0]
247
+ target_height = image_size * target_aspect_ratio[1]
248
+ blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
249
+
250
+ # resize the image
251
+ resized_img = image.resize((target_width, target_height))
252
+ processed_images = []
253
+ for i in range(blocks):
254
+ box = (
255
+ (i % (target_width // image_size)) * image_size,
256
+ (i // (target_width // image_size)) * image_size,
257
+ ((i % (target_width // image_size)) + 1) * image_size,
258
+ ((i // (target_width // image_size)) + 1) * image_size
259
+ )
260
+ # split the image
261
+ split_img = resized_img.crop(box)
262
+ processed_images.append(split_img)
263
+ assert len(processed_images) == blocks
264
+ if use_thumbnail and len(processed_images) != 1:
265
+ thumbnail_img = image.resize((image_size, image_size))
266
+ processed_images.append(thumbnail_img)
267
+ return processed_images, target_aspect_ratio
268
+
269
+
270
+
271
+ def normalize_transform(mean, std):
272
+ if mean is None and std is None:
273
+ transform = None
274
+ elif mean is None and std is not None:
275
+ mean = [0.] * len(std)
276
+ transform = transforms.Normalize(mean=mean, std=std)
277
+ elif mean is not None and std is None:
278
+ std = [1.] * len(mean)
279
+ transform = transforms.Normalize(mean=mean, std=std)
280
+ else:
281
+ transform = transforms.Normalize(mean=mean, std=std)
282
+
283
+ return transform
284
+
285
+
286
+
287
+ def format_messages(
288
+ conversations: List[Dict[str, str]],
289
+ sft_format: str = "deepseek",
290
+ system_prompt: str = "",
291
+ ):
292
+ """
293
+ Applies the SFT template to conversation.
294
+
295
+ Args:
296
+ conversations (List[Dict]): A List of messages.
297
+ sft_format (str, optional): The format of the SFT template to use. Defaults to "deepseek".
298
+ system_prompt (str, optional): The system prompt to use in the SFT template. Defaults to "".
299
+
300
+ Returns:
301
+ sft_prompt (str): The formatted text.
302
+ """
303
+
304
+ conv = get_conv_template(sft_format)
305
+ conv.set_system_message(system_prompt)
306
+ for message in conversations:
307
+ conv.append_message(message["role"], message["content"].strip())
308
+ sft_prompt = conv.get_prompt().strip()
309
+
310
+ return sft_prompt
311
+
312
+
313
+ def text_encode(tokenizer, text: str, bos: bool = True, eos: bool = False):
314
+ t = tokenizer.encode(text, add_special_tokens=False)
315
+ bos_id = 0
316
+ eos_id = 1
317
+ if bos:
318
+ t = [bos_id] + t
319
+ if eos:
320
+ t = t + [eos_id]
321
+
322
+ return t
323
+
324
+ def load_pil_images(conversations: List[Dict[str, str]]) -> List[Image.Image]:
325
+ """
326
+
327
+ Args:
328
+ conversations (List[Dict[str, str]]): the conversations with a list of messages. An example is :
329
+ [
330
+ {
331
+ "role": "User",
332
+ "content": "<image_placeholder>\nExtract all information from this image and convert them into markdown format.",
333
+ "images": ["./examples/table_datasets.png"]
334
+ },
335
+ {"role": "Assistant", "content": ""},
336
+ ]
337
+
338
+ Returns:
339
+ pil_images (List[PIL.Image.Image]): the list of PIL images.
340
+
341
+ """
342
+
343
+ pil_images = []
344
+
345
+ for message in conversations:
346
+ if "images" not in message:
347
+ continue
348
+
349
+ for image_path in message["images"]:
350
+ # print('----------------')
351
+ # print(image_path)
352
+ # print('----------------')
353
+ # exit()
354
+
355
+ # pil_img = Image.open(image_path)
356
+ pil_img = load_image(image_path)
357
+ pil_img = pil_img.convert("RGB")
358
+ pil_images.append(pil_img)
359
+
360
+ return pil_images
361
+
362
+
363
+ class BaseTransform(ABC):
364
+
365
+ def set_rng(self, *args, **kwargs):
366
+ pass
367
+
368
+ def __call__(self, *args, **kwargs) -> torch.Tensor:
369
+ pass
370
+
371
+ @property
372
+ def default_shape(self):
373
+ raise NotImplementedError
374
+
375
+
376
+ class BasicImageTransform(BaseTransform):
377
+ def __init__(
378
+ self,
379
+ mean: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5),
380
+ std: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5),
381
+ normalize: bool = True
382
+ ):
383
+ self.mean = mean
384
+ self.std = std
385
+
386
+ transform_pipelines = [
387
+ transforms.ToTensor()
388
+ ]
389
+
390
+ normalize = normalize_transform(mean, std) if normalize else nn.Identity()
391
+ if normalize is not None:
392
+ transform_pipelines.append(normalize)
393
+
394
+ self.transform = transforms.Compose(transform_pipelines)
395
+
396
+ def __call__(self, x):
397
+ x = self.transform(x)
398
+ return x
399
+
400
+ class NoEOSTextStreamer(TextStreamer):
401
+ def on_finalized_text(self, text: str, stream_end: bool = False):
402
+
403
+ eos_text = self.tokenizer.decode([self.tokenizer.eos_token_id], skip_special_tokens=False)
404
+ text = text.replace(eos_text, "\n")
405
+ print(text, flush=True, end="")
406
+
407
+
408
+ class SlidingWindowNoRepeatNgramProcessor:
409
+ """Block n-gram repetitions within a sliding window.
410
+ Aligned with SGLang DeepseekOCRNoRepeatNGramLogitProcessor."""
411
+ def __init__(self, ngram_size, window, whitelist_token_ids=None):
412
+ self.ngram_size = ngram_size
413
+ self.window = window
414
+ self.whitelist = set(whitelist_token_ids) if whitelist_token_ids else set()
415
+
416
+ def __call__(self, input_ids, scores):
417
+ for batch_idx in range(input_ids.shape[0]):
418
+ sequence = input_ids[batch_idx].tolist()
419
+ if len(sequence) < self.ngram_size:
420
+ continue
421
+ search_start = max(0, len(sequence) - self.window)
422
+ search_end = len(sequence) - self.ngram_size + 1
423
+ if search_end <= search_start:
424
+ continue
425
+ if self.ngram_size > 1:
426
+ current_prefix = tuple(sequence[-(self.ngram_size - 1):])
427
+ else:
428
+ current_prefix = tuple()
429
+ banned = set()
430
+ for idx in range(search_start, search_end):
431
+ ngram = sequence[idx:idx + self.ngram_size]
432
+ if self.ngram_size == 1 or tuple(ngram[:-1]) == current_prefix:
433
+ banned.add(ngram[-1])
434
+ banned.difference_update(self.whitelist)
435
+ for token_id in banned:
436
+ scores[batch_idx, token_id] = float('-inf')
437
+ return scores
438
+
439
+
440
+ class TPSTextStreamer(TextStreamer):
441
+ """Streamer that prints TPS every `interval` tokens. Set interval=0 to disable."""
442
+ def __init__(self, tokenizer, interval=100, **kwargs):
443
+ super().__init__(tokenizer, **kwargs)
444
+ self.interval = interval
445
+ self.token_count = 0
446
+ self.start_time = None
447
+ self.start_token_count = 0
448
+ self.last_report_count = 0
449
+ self.last_report_time = None
450
+
451
+ def put(self, value):
452
+ import time
453
+ if hasattr(value, 'numel'):
454
+ self.token_count += value.numel()
455
+ else:
456
+ self.token_count += 1
457
+ if self.start_time is None:
458
+ self.start_time = time.time()
459
+ self.last_report_time = self.start_time
460
+ self.last_report_count = self.token_count
461
+ self.start_token_count = self.token_count
462
+ super().put(value)
463
+ return
464
+ if self.interval > 0 and self.token_count - self.last_report_count >= self.interval:
465
+ now = time.time()
466
+ delta_tokens = self.token_count - self.last_report_count
467
+ delta_time = now - self.last_report_time
468
+ recent_tps = delta_tokens / delta_time if delta_time > 0 else 0
469
+ avg_tps = (self.token_count - self.start_token_count) / (now - self.start_time) if (now - self.start_time) > 0 else 0
470
+ print(f"\n[TPS] tokens={self.token_count}, recent={recent_tps:.1f} t/s, avg={avg_tps:.1f} t/s", flush=True)
471
+ self.last_report_count = self.token_count
472
+ self.last_report_time = now
473
+ super().put(value)
474
+
475
+ def on_finalized_text(self, text: str, stream_end: bool = False):
476
+ eos_text = self.tokenizer.decode([self.tokenizer.eos_token_id], skip_special_tokens=False)
477
+ text = text.replace(eos_text, "\n")
478
+ print(text, flush=True, end="")
479
+
480
+
481
+ @torch.no_grad()
482
+ def _uocr_dash_decode_allow(indexer, input_ids, cache):
483
+ """Page-aware remote candidate blocks of the DECODE query (vLLM page_state.py parity).
484
+
485
+ Prefill (q_len > 1): record, per KV block, the [min, max] image page it holds, the
486
+ prompt page count ``pp`` (<PAGE> markers up to the last image token) and the marker
487
+ count. Returns None (prefill builds its own per-query candidates).
488
+ Decode (q_len == 1): the query token's page is ``marker_cum - 1 - pp`` (response pages
489
+ restart at 0, so text page k <-> image page k); the candidate set is the blocks holding
490
+ an image token of that page, or every block when the page is unknown / has no image
491
+ block (training's per-query fallback). Returns [B, nblk] bool.
492
+ """
493
+ img_id, pg_id = indexer.image_token_id, indexer.page_token_id
494
+ if img_id is None or pg_id is None:
495
+ return None
496
+ bs = int(indexer.block_size)
497
+ ids = input_ids
498
+ B, L = ids.shape
499
+ if L > 1:
500
+ img = ids == img_id
501
+ cum = torch.cumsum((ids == pg_id).long(), dim=-1)
502
+ run = torch.cumsum(img.long(), dim=-1)
503
+ per_page = max(1, int(indexer.per_page_image_tokens))
504
+ page = torch.where(cum > 0, cum - 1, (run - 1) // per_page)
505
+ page = torch.where(img, page, torch.full_like(page, -1))
506
+ pp = torch.where(img, cum, torch.zeros_like(cum)).cummax(dim=-1).values[:, -1]
507
+ nblk = -(-L // bs)
508
+ pg = F.pad(page, (0, nblk * bs - L), value=-1).view(B, nblk, bs)
509
+ bmax = pg.amax(dim=-1)
510
+ bmin = torch.where(pg >= 0, pg, torch.full_like(pg, 1 << 30)).amin(dim=-1)
511
+ bmin = torch.where(bmax >= 0, bmin, torch.full_like(bmin, -1))
512
+ cache._uocr_pg = {'bmin': bmin, 'bmax': bmax, 'pp': pp, 'mcum': cum[:, -1].clone()}
513
+ return None
514
+ st = getattr(cache, '_uocr_pg', None)
515
+ if st is None:
516
+ return None
517
+ st['mcum'] = st['mcum'] + (ids[:, -1] == pg_id).long()
518
+ qp = st['mcum'] - 1 - st['pp']
519
+ qp = torch.where(qp >= 0, qp, torch.full_like(qp, -1))[:, None]
520
+ total = int(cache.get_seq_length()) + L
521
+ nblk = -(-total // bs)
522
+ bmin, bmax = st['bmin'], st['bmax']
523
+ same = (bmin >= 0) & (bmin <= qp) & (bmax >= qp) & (qp >= 0)
524
+ open_row = (qp[:, 0] < 0) | ~same.any(dim=-1)
525
+ allow = torch.zeros(B, nblk, dtype=torch.bool, device=ids.device)
526
+ n0 = min(nblk, same.shape[-1])
527
+ allow[:, :n0] = same[:, :n0]
528
+ return allow | open_row[:, None]
529
+
530
+
531
+ def _uocr_build_inputs(tokenizer, prompt, images, crop_mode=None, base_size=1024, image_size=640,
532
+ min_crops=2, max_crops=16, patch_size=16, downsample_ratio=4,
533
+ image_token='<image>', image_token_id=128815, bos_id=0):
534
+ """Tokens + pixel tensors exactly like vLLM's UnlimitedOCRProcessor.tokenize_with_images.
535
+
536
+ crop_mode None == vLLM auto policy: gundam (dynamic tiles) for a single image, base
537
+ (one letterbox-padded global view) for multi-image. No 640x640 pre-resize in base mode
538
+ (vLLM's training-parity fix): the original page is padded straight into base_size.
539
+ """
540
+ n_img = len(images)
541
+ cropping = (n_img == 1) if crop_mode is None else bool(crop_mode)
542
+ splits = prompt.split(image_token)
543
+ assert len(splits) == n_img + 1, f'prompt has {len(splits) - 1} {image_token} but {n_img} images'
544
+ transform = BasicImageTransform(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5), normalize=True)
545
+ pad_color = tuple(int(x * 255) for x in transform.mean)
546
+ nq = math.ceil((image_size // patch_size) / downsample_ratio)
547
+ nqb = math.ceil((base_size // patch_size) / downsample_ratio)
548
+ ids, seq_mask, globals_, crops, spatial = [], [], [], [], []
549
+ for text_sep, image in zip(splits, images):
550
+ t = tokenizer.encode(text_sep, add_special_tokens=False)
551
+ ids += t
552
+ seq_mask += [False] * len(t)
553
+ tiles = []
554
+ if image.size[0] <= image_size and image.size[1] <= image_size:
555
+ ratio = (1, 1)
556
+ elif cropping:
557
+ tiles, ratio = dynamic_preprocess(image, min_num=min_crops, max_num=max_crops,
558
+ image_size=image_size)
559
+ else:
560
+ ratio = (1, 1)
561
+ globals_.append(transform(ImageOps.pad(image, (base_size, base_size), color=pad_color)).to(torch.bfloat16))
562
+ w_t, h_t = int(ratio[0]), int(ratio[1])
563
+ spatial.append([w_t, h_t])
564
+ tok = [image_token_id] * (nqb * (nqb + 1)) + [image_token_id]
565
+ if w_t > 1 or h_t > 1:
566
+ crops += [transform(c).to(torch.bfloat16) for c in tiles]
567
+ tok += ([image_token_id] * (nq * w_t + 1)) * (nq * h_t)
568
+ ids += tok
569
+ seq_mask += [True] * len(tok)
570
+ t = tokenizer.encode(splits[-1], add_special_tokens=False)
571
+ ids = [bos_id] + ids + t
572
+ seq_mask = [False] + seq_mask + [False] * len(t)
573
+ input_ids = torch.LongTensor(ids)
574
+ seq_mask = torch.tensor(seq_mask, dtype=torch.bool)
575
+ if n_img == 0:
576
+ return input_ids, None, seq_mask, None
577
+ images_ori = torch.stack(globals_, dim=0)
578
+ images_crop = (torch.stack(crops, dim=0) if crops
579
+ else torch.zeros((1, 3, base_size, base_size), dtype=torch.bfloat16))
580
+ return input_ids, (images_crop, images_ori), seq_mask, torch.tensor(spatial, dtype=torch.long)
581
+
582
+
583
+ def _uocr_page_count_guard(logits, gen_ids, expected_pages, page_id, eos_id, max_page_tokens, st):
584
+ """vLLM unlimited_sparse_plugin.page_count_guard._PageCountGuard (per-step, in place)."""
585
+ if expected_pages <= 0:
586
+ return logits
587
+ n = len(gen_ids)
588
+ if n > st['scanned']:
589
+ tail = gen_ids[st['scanned']:n]
590
+ m = tail.count(page_id)
591
+ if m:
592
+ st['marker_cum'] += m
593
+ st['page_len'] = tail[::-1].index(page_id)
594
+ else:
595
+ st['page_len'] += len(tail)
596
+ st['scanned'] = n
597
+ mc = st['marker_cum']
598
+ if mc > expected_pages:
599
+ keep = logits[eos_id].clone(); logits[:] = -float('inf'); logits[eos_id] = keep
600
+ elif 0 < max_page_tokens <= st['page_len']:
601
+ tid = page_id if mc < expected_pages else eos_id
602
+ keep = logits[tid].clone(); logits[:] = -float('inf'); logits[tid] = keep
603
+ elif mc < expected_pages:
604
+ logits[eos_id] = -float('inf')
605
+ return logits
606
+
607
+
608
+ def _uocr_no_repeat_ngram(logits, gen_ids, ngram_size, window_size, whitelist):
609
+ """vLLM deepseek_ocr.NoRepeatNGramLogitsProcessor (on generated ids only)."""
610
+ if ngram_size <= 0 or len(gen_ids) < ngram_size:
611
+ return logits
612
+ prefix = tuple(gen_ids[-(ngram_size - 1):]) if ngram_size > 1 else tuple()
613
+ start = max(0, len(gen_ids) - window_size)
614
+ end = len(gen_ids) - ngram_size + 1
615
+ banned = set()
616
+ for i in range(start, end):
617
+ ng = tuple(gen_ids[i:i + ngram_size])
618
+ if ng[:-1] == prefix:
619
+ banned.add(ng[-1])
620
+ banned -= whitelist
621
+ if banned:
622
+ logits[list(banned)] = -float('inf')
623
+ return logits
624
+
625
+
626
+ class UnlimitedOCRConfig(DeepseekV2Config):
627
+ model_type = "unlimited-ocr"
628
+
629
+ class UnlimitedOCRModel(DeepseekV2Model):
630
+ config_class = UnlimitedOCRConfig
631
+
632
+ def __init__(self, config: DeepseekV2Config):
633
+ super(UnlimitedOCRModel, self).__init__(config)
634
+
635
+ self.sam_model = build_sam_vit_b(config=config)
636
+ self.vision_model = build_clip_l(config=config)
637
+ # self.conv_2 = nn.Conv2d(in_channels=1024, out_channels=2048, kernel_size=2, stride=2)
638
+ n_embed = config.hidden_size
639
+ self.projector = MlpProjector(Dict(projector_type="linear", input_dim=config.projector_config["input_dim"], n_embed=n_embed))
640
+ embed_std = 1 / torch.sqrt(torch.tensor(n_embed, dtype=torch.float32))
641
+ self.image_newline = nn.Parameter(torch.randn(n_embed) * embed_std)
642
+ self.view_seperator = nn.Parameter(torch.randn(n_embed) * embed_std)
643
+
644
+
645
+
646
+
647
+ def forward(
648
+ self,
649
+ input_ids: torch.LongTensor = None,
650
+ attention_mask: Optional[torch.Tensor] = None,
651
+ position_ids: Optional[torch.LongTensor] = None,
652
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
653
+ inputs_embeds: Optional[torch.FloatTensor] = None,
654
+ use_cache: Optional[bool] = None,
655
+ output_attentions: Optional[bool] = None,
656
+ output_hidden_states: Optional[bool] = None,
657
+ images: Optional[torch.FloatTensor] = None,
658
+ images_seq_mask: Optional[torch.FloatTensor] = None,
659
+ images_spatial_crop: Optional[torch.FloatTensor] = None,
660
+ return_dict: Optional[bool] = None,
661
+ ) -> Union[Tuple, BaseModelOutputWithPast]:
662
+
663
+ # hand the full token id sequence directly to each sparse attention module so the
664
+ # indexer can derive per-page image/query ids (page-aware image bucket constraint).
665
+ # Set on the attention module itself (not config) to avoid any config-object identity
666
+ # ambiguity and to survive gradient-checkpointing recompute.
667
+ if input_ids is not None and getattr(input_ids, 'ndim', 0) == 2:
668
+ _rope2d = None
669
+ _layers = list(self.layers)
670
+ _first = next((getattr(l, 'self_attn', None) for l in _layers
671
+ if getattr(getattr(l, 'self_attn', None), 'indexer', None) is not None),
672
+ None)
673
+ if (_first is not None and input_ids.shape[-1] > 1
674
+ and getattr(_first, 'main_rope_mode', '1d') == 'page2d'):
675
+ _rope2d = _first.indexer._build_page_rope_pos(input_ids, input_ids.shape[-1])
676
+ # The per-page prefill counts (pf_cnt) that seed the decode-side page-rope state
677
+ # machine (_main_rope_coords) are written only on the indexer that built the
678
+ # coordinates (layer 0); hand them to every layer, otherwise layers 1.. fall
679
+ # back to 1D RoPE as soon as decoding starts.
680
+ _meta = (getattr(_first.indexer, '_page_rope_meta', None)
681
+ if (_first is not None and _rope2d is not None) else None)
682
+ # infer(): page-aware candidate blocks of the decode query (vLLM page_state parity).
683
+ _dec_allow = None
684
+ if _DASH_INFER["on"] and past_key_values is not None and _first is not None:
685
+ _dec_allow = _uocr_dash_decode_allow(_first.indexer, input_ids, past_key_values)
686
+ for _layer in _layers:
687
+ _attn = getattr(_layer, 'self_attn', None)
688
+ if _attn is not None and getattr(_attn, 'indexer', None) is not None:
689
+ _attn._sparse_source_input_ids = input_ids
690
+ _attn._main_rope_pos_2d = _rope2d
691
+ if _meta is not None:
692
+ _attn.indexer._page_rope_meta = _meta
693
+ _attn._dash_dec_allow = _dec_allow
694
+
695
+ if inputs_embeds is None:
696
+ # inputs_embeds = self.embed_tokens(input_ids)
697
+ inputs_embeds = self.get_input_embeddings()(input_ids)
698
+
699
+
700
+
701
+ sam_model = getattr(self, 'sam_model', None)
702
+ # sam_model = self.sam_model
703
+ vision_model = getattr(self, 'vision_model', None)
704
+
705
+
706
+
707
+ # TIME_DIAG=1 -> attribute the per-page vision encoder cost.
708
+ _t_vis = _td_now()
709
+ if sam_model is not None and images is not None and (input_ids.shape[1] != 1 or self.training) and torch.sum(images[0][1]).item() != 0:
710
+
711
+ _packed_mm = (inputs_embeds.shape[0] == 1 and len(images) > 1)
712
+ _packed_feats = []
713
+ for idx, image in enumerate(images):
714
+ images_in_this_batch = []
715
+ patches_all = image[0]
716
+ image_ori = image[1]
717
+ spatial = image[2] if len(image) > 2 else None
718
+
719
+ num_pages = image_ori.shape[0]
720
+ tile_cursor = 0
721
+ with torch.no_grad():
722
+ for page_idx in range(num_pages):
723
+ image_ori_p = image_ori[page_idx:page_idx + 1]
724
+ if spatial is not None:
725
+ width_crop_num = int(spatial[page_idx][0])
726
+ height_crop_num = int(spatial[page_idx][1])
727
+ else:
728
+ width_crop_num = height_crop_num = 1
729
+ do_crop = (width_crop_num > 1 or height_crop_num > 1) and torch.sum(patches_all).item() != 0
730
+
731
+ if do_crop:
732
+ n_tiles = width_crop_num * height_crop_num
733
+ patches = patches_all[tile_cursor:tile_cursor + n_tiles]
734
+ tile_cursor += n_tiles
735
+ local_features_1 = sam_model(patches)
736
+ local_features_2 = vision_model(patches, local_features_1)
737
+ local_features = torch.cat((local_features_2[:, 1:], local_features_1.flatten(2).permute(0, 2, 1)), dim=-1)
738
+ local_features = self.projector(local_features)
739
+
740
+ global_features_1 = sam_model(image_ori_p)
741
+ global_features_2 = vision_model(image_ori_p, global_features_1)
742
+ global_features = torch.cat((global_features_2[:, 1:], global_features_1.flatten(2).permute(0, 2, 1)), dim=-1)
743
+ global_features = self.projector(global_features)
744
+
745
+ _, hw, n_dim = global_features.shape
746
+ h = w = int(hw ** 0.5)
747
+ _2, hw2, n_dim2 = local_features.shape
748
+ h2 = w2 = int(hw2 ** 0.5)
749
+
750
+ global_features = global_features.view(h, w, n_dim)
751
+ global_features = torch.cat(
752
+ [global_features, self.image_newline[None, None, :].expand(h, 1, n_dim)], dim=1
753
+ )
754
+ global_features = global_features.view(-1, n_dim)
755
+
756
+ local_features = local_features.view(height_crop_num, width_crop_num, h2, w2, n_dim2).permute(0, 2, 1, 3, 4).reshape(height_crop_num * h2, width_crop_num * w2, n_dim2)
757
+ local_features = torch.cat(
758
+ [local_features, self.image_newline[None, None, :].expand(height_crop_num * h2, 1, n_dim2)], dim=1
759
+ )
760
+ local_features = local_features.view(-1, n_dim2)
761
+
762
+ global_local_features = torch.cat([local_features, global_features, self.view_seperator[None, :]], dim=0)
763
+ images_in_this_batch.append(global_local_features)
764
+ else:
765
+ global_features_1 = sam_model(image_ori_p)
766
+ global_features_2 = vision_model(image_ori_p, global_features_1)
767
+ global_features = torch.cat((global_features_2[:, 1:], global_features_1.flatten(2).permute(0, 2, 1)), dim=-1)
768
+ global_features = self.projector(global_features)
769
+
770
+ _, hw, n_dim = global_features.shape
771
+ h = w = int(hw ** 0.5)
772
+ global_features = global_features.view(h, w, n_dim)
773
+ global_features = torch.cat(
774
+ [global_features, self.image_newline[None, None, :].expand(h, 1, n_dim)], dim=1
775
+ )
776
+ global_features = global_features.view(-1, n_dim)
777
+ global_local_features = torch.cat([global_features, self.view_seperator[None, :]], dim=0)
778
+ images_in_this_batch.append(global_local_features)
779
+
780
+ if images_in_this_batch:
781
+ images_in_this_batch = torch.cat(images_in_this_batch, dim=0)
782
+ if _packed_mm:
783
+ _packed_feats.append(images_in_this_batch)
784
+ else:
785
+ inputs_embeds[idx].masked_scatter_(images_seq_mask[idx].unsqueeze(-1).cuda(), images_in_this_batch)
786
+
787
+ if _packed_mm and _packed_feats:
788
+ _packed_feats = torch.cat(_packed_feats, dim=0)
789
+ inputs_embeds[0].masked_scatter_(images_seq_mask[0].unsqueeze(-1).cuda(), _packed_feats)
790
+
791
+ _td_add("vision", _t_vis)
792
+
793
+ with _td("llm_decoder"):
794
+ return super(UnlimitedOCRModel, self).forward(
795
+ input_ids=None, attention_mask=attention_mask, past_key_values=past_key_values,
796
+ inputs_embeds=inputs_embeds, use_cache=use_cache, position_ids = position_ids,
797
+ output_attentions=output_attentions, output_hidden_states=output_hidden_states,
798
+ return_dict=return_dict
799
+ )
800
+
801
+
802
+ class UnlimitedOCRForCausalLM(DeepseekV2ForCausalLM):
803
+
804
+ config_class = UnlimitedOCRConfig
805
+ # supports_gradient_checkpointing = True
806
+
807
+ def __init__(self, config):
808
+ super(DeepseekV2ForCausalLM, self).__init__(config)
809
+ self.model = UnlimitedOCRModel(config)
810
+
811
+ self.vocab_size = config.vocab_size
812
+
813
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
814
+
815
+ # self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
816
+
817
+ # Initialize weights and apply final processing
818
+ self.post_init()
819
+
820
+ def get_model(self):
821
+ return self.model
822
+
823
+
824
+ def forward(
825
+ self,
826
+ input_ids: torch.LongTensor = None,
827
+ attention_mask: Optional[torch.Tensor] = None,
828
+ position_ids: Optional[torch.LongTensor] = None,
829
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
830
+ inputs_embeds: Optional[torch.FloatTensor] = None,
831
+ labels: Optional[torch.LongTensor] = None,
832
+ use_cache: Optional[bool] = None,
833
+ output_attentions: Optional[bool] = None,
834
+ output_hidden_states: Optional[bool] = None,
835
+ images: Optional[torch.FloatTensor] = None,
836
+ images_seq_mask: Optional[torch.FloatTensor] = None,
837
+ images_spatial_crop: Optional[torch.FloatTensor] = None,
838
+ return_dict: Optional[bool] = None,
839
+
840
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
841
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
842
+ output_hidden_states = (
843
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
844
+ )
845
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
846
+
847
+
848
+
849
+ outputs = self.model(
850
+ input_ids=input_ids,
851
+ past_key_values=past_key_values,
852
+ attention_mask=attention_mask,
853
+ position_ids=position_ids,
854
+ inputs_embeds=inputs_embeds,
855
+ use_cache=use_cache,
856
+ output_attentions=output_attentions,
857
+ output_hidden_states=output_hidden_states,
858
+ images=images,
859
+ images_seq_mask = images_seq_mask,
860
+ images_spatial_crop = images_spatial_crop,
861
+ return_dict=return_dict
862
+
863
+ )
864
+
865
+
866
+
867
+ # print(transformer_outputs)
868
+
869
+ hidden_states = outputs[0]
870
+
871
+ # fused chunked linear cross-entropy. Avoids materialising the full
872
+ # [B, seq, vocab=129280] logits (and its fp32 upcast) that OOMs on long OCR sequences.
873
+ # Gated by FUSED_CHUNK_CE (default on when labels present); falls back to the classic
874
+ # full-logits path so generation / return-logits callers stay unaffected.
875
+ _use_fused_ce = (
876
+ labels is not None
877
+ and os.environ.get('FUSED_CHUNK_CE', '1') == '1'
878
+ )
879
+
880
+ logits = None
881
+ loss = None
882
+ token_acc_correct = None
883
+ token_acc_valid = None
884
+ if _use_fused_ce:
885
+ with _td("ce"):
886
+ loss, token_acc_correct, token_acc_valid = _fused_chunked_lm_ce(
887
+ hidden_states,
888
+ self.lm_head,
889
+ labels,
890
+ self.config.vocab_size,
891
+ num_items_in_batch=None,
892
+ chunk_size=int(os.environ.get('CE_CHUNK_SIZE', '1024')),
893
+ return_token_acc=True,
894
+ )
895
+ _td_report(f"seq={hidden_states.shape[1]}")
896
+ else:
897
+ logits = self.lm_head(hidden_states)
898
+ logits = logits.float()
899
+
900
+ if labels is not None:
901
+ # Shift so that tokens < n predict n
902
+ shift_logits = logits[..., :-1, :].contiguous()
903
+ shift_labels = labels[..., 1:].contiguous()
904
+ # Flatten the tokens
905
+ loss_fct = CrossEntropyLoss()
906
+ shift_logits = shift_logits.view(-1, self.config.vocab_size)
907
+ shift_labels = shift_labels.view(-1)
908
+ # Enable model parallelism
909
+ shift_labels = shift_labels.to(shift_logits.device)
910
+ loss = loss_fct(shift_logits, shift_labels)
911
+
912
+ # propagate indexer_kl_loss to outer output; trainer is responsible for
913
+ # weighting/adding it into the final loss and logging. Do NOT add KL here to
914
+ # avoid double-counting when swift's compute_loss picks it up.
915
+ _kl = getattr(outputs, 'indexer_kl_loss', None)
916
+ # one-shot diagnostic
917
+ if not getattr(self, '_kl_prop_debug_logged', False):
918
+ self._kl_prop_debug_logged = True
919
+ try:
920
+ _inner_type = type(outputs).__name__
921
+ _kl_repr = 'None' if _kl is None else f'tensor(shape={list(_kl.shape)}, dtype={_kl.dtype}, val={float(_kl.detach()):.6f})'
922
+ print(f"[debug ForCausalLM] inner_outputs_type={_inner_type} "
923
+ f"inner_has_indexer_kl={hasattr(outputs, 'indexer_kl_loss')} "
924
+ f"_kl={_kl_repr} labels_not_None={labels is not None}", flush=True)
925
+ except Exception as _e:
926
+ print(f"[debug ForCausalLM] failed: {_e}", flush=True)
927
+
928
+ if not return_dict:
929
+ output = (logits,) + outputs[1:]
930
+ return (loss,) + output if loss is not None else output
931
+
932
+ result = CausalLMOutputWithPastAndKL(
933
+ loss=loss,
934
+ logits=logits,
935
+ past_key_values=outputs.past_key_values,
936
+ hidden_states=outputs.hidden_states,
937
+ attentions=outputs.attentions,
938
+ indexer_kl_loss=_kl, # proper dataclass field, survives DDP pytree
939
+ token_acc_correct=token_acc_correct, # fused-CE correct token count
940
+ token_acc_valid=token_acc_valid, # fused-CE valid token count
941
+ )
942
+ return result
943
+
944
+
945
+ def prepare_inputs_for_generation(
946
+ self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
947
+ ):
948
+ # Omit tokens covered by past_key_values
949
+ past_length = 0
950
+ if past_key_values is not None:
951
+ if isinstance(past_key_values, Cache):
952
+ cache_length = past_key_values.get_seq_length()
953
+ past_length = past_key_values.get_seq_length()
954
+ max_cache_length = getattr(past_key_values, 'get_max_length', lambda: None)()
955
+ else:
956
+ cache_length = past_length = past_key_values[0][0].shape[2]
957
+ max_cache_length = None
958
+
959
+ # Ring buffer: cache size is fixed, but we've processed more tokens.
960
+ # Always just take the last token for decode.
961
+ if hasattr(past_key_values, '_prefill_length') and past_length > 0:
962
+ input_ids = input_ids[:, -1:]
963
+ # Keep only the unprocessed tokens:
964
+ # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
965
+ # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as
966
+ # input)
967
+ elif attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
968
+ input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
969
+ # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
970
+ # input_ids based on the past_length.
971
+ elif past_length < input_ids.shape[1]:
972
+ input_ids = input_ids[:, past_length:]
973
+ # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
974
+
975
+ # If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
976
+ if (
977
+ max_cache_length is not None
978
+ and attention_mask is not None
979
+ and cache_length + input_ids.shape[1] > max_cache_length
980
+ ):
981
+ attention_mask = attention_mask[:, -max_cache_length:]
982
+
983
+ position_ids = kwargs.get("position_ids", None)
984
+ if attention_mask is not None and position_ids is None:
985
+ # create position_ids on the fly for batch generation
986
+ position_ids = attention_mask.long().cumsum(-1) - 1
987
+ position_ids.masked_fill_(attention_mask == 0, 1)
988
+ if past_key_values:
989
+ position_ids = position_ids[:, -input_ids.shape[1] :]
990
+
991
+ # if self.generation_config.cache_implementation == "static":
992
+ # # generation with static cache
993
+ # cache_position = kwargs.get("cache_position", None)
994
+ # if cache_position is None:
995
+ # past_length = 0
996
+ # else:
997
+ # past_length = cache_position[-1] + 1
998
+ # input_ids = input_ids[:, past_length:]
999
+ # position_ids = position_ids[:, past_length:]
1000
+
1001
+ # TODO @gante we should only keep a `cache_position` in generate, and do +=1.
1002
+ # same goes for position ids. Could also help with continued generation.
1003
+ cache_position = torch.arange(past_length, past_length + position_ids.shape[-1], device=position_ids.device)
1004
+
1005
+ # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
1006
+ if inputs_embeds is not None and past_key_values is None:
1007
+ model_inputs = {"inputs_embeds": inputs_embeds}
1008
+ else:
1009
+ model_inputs = {"input_ids": input_ids}
1010
+
1011
+ # Pass images only on prefill (cache empty or None)
1012
+ _is_prefill = (past_key_values is None or
1013
+ (isinstance(past_key_values, Cache) and past_key_values.get_seq_length() == 0))
1014
+ model_inputs.update(
1015
+ {
1016
+ "position_ids": position_ids,
1017
+ "past_key_values": past_key_values,
1018
+ "use_cache": kwargs.get("use_cache"),
1019
+ "attention_mask": attention_mask,
1020
+ "images": kwargs.get("images", None) if _is_prefill else None,
1021
+ "images_seq_mask": kwargs.get("images_seq_mask", None) if _is_prefill else None,
1022
+ "images_spatial_crop": kwargs.get("images_spatial_crop", None) if _is_prefill else None,
1023
+ }
1024
+ )
1025
+ return model_inputs
1026
+
1027
+
1028
+ def disable_torch_init(self):
1029
+ """
1030
+ Disable the redundant torch default initialization to accelerate model creation.
1031
+ """
1032
+ import torch
1033
+ setattr(torch.nn.Linear, "reset_parameters", lambda self: None)
1034
+ setattr(torch.nn.LayerNorm, "reset_parameters", lambda self: None)
1035
+
1036
+
1037
+
1038
+ @torch.no_grad()
1039
+ def infer(self, tokenizer, prompt='', image_file='', output_path='', base_size=1024, image_size=640,
1040
+ crop_mode=None, test_compress=False, save_results=False, eval_mode=False, max_length=131072,
1041
+ max_new_tokens=None, tps_interval=0, no_repeat_ngram_size=0, ngram_window=0,
1042
+ ngram_whitelist_token_ids=None, temperature=0.0, page_count_guard=True,
1043
+ max_page_tokens=20000, dash_topm=16, max_crops=16, return_dict=False):
1044
+ """Single-/multi-page inference, aligned with the vLLM deployment
1045
+ (sparse_attn_infer-451/deploy_multimodel_dash_01.sh + request_openai*.py).
1046
+
1047
+ image_file: one path / PIL image, or a list of them (one per page, in order).
1048
+ prompt: instruction text. Without ``<image>`` placeholders the trained multi-page
1049
+ layout is built: ``"<PAGE><image>" * n_pages + prompt`` (default prompt
1050
+ "Multi page parsing."). A prompt with exactly n_pages ``<image>`` is used as is;
1051
+ a single ``<image>`` with several pages is expanded to ``<PAGE><image>`` per page.
1052
+ crop_mode: None -> vLLM auto (gundam tiles for one page, base for several);
1053
+ True / False forces gundam / base.
1054
+ Generation mirrors vLLM: greedy when temperature == 0, stops at EOS or
1055
+ ``max_new_tokens`` (default ``max_length - prompt_len``), PageCountGuard on by default
1056
+ (prompt <PAGE> count == output pages; per-page cap ``max_page_tokens``), optional
1057
+ NoRepeatNGram (``no_repeat_ngram_size`` / ``ngram_window``, vLLM extra_args semantics).
1058
+ Decoder attention = DASH block-sparse in prefill and decode (vLLM DASH_DECODE_DENSE=0,
1059
+ DASH_INFER_TOPM=dash_topm, DASH_PAGE_VISIBLE_BACK=0).
1060
+
1061
+ Returns the generated text (special tokens kept, EOS dropped -- like vLLM with
1062
+ skip_special_tokens=False), or a dict with token ids when ``return_dict``.
1063
+ """
1064
+ if isinstance(image_file, (list, tuple)):
1065
+ files = [f for f in image_file if f is not None and f != '']
1066
+ elif image_file is None or image_file == '':
1067
+ files = []
1068
+ else:
1069
+ files = [image_file]
1070
+ images = [f.convert('RGB') if isinstance(f, Image.Image) else Image.open(f).convert('RGB')
1071
+ for f in files]
1072
+ n_img = len(images)
1073
+ prompt = prompt or ('Multi page parsing.' if n_img else '')
1074
+ if not prompt:
1075
+ raise ValueError('prompt is empty!')
1076
+ n_ph = prompt.count('<image>')
1077
+ if n_img and n_ph == 0:
1078
+ prompt = '<PAGE><image>' * n_img + prompt
1079
+ elif n_img > 1 and n_ph == 1:
1080
+ prompt = prompt.replace('<image>', '<PAGE><image>' * n_img)
1081
+ elif n_ph != n_img:
1082
+ raise ValueError(f'prompt has {n_ph} <image> placeholders but {n_img} images')
1083
+
1084
+ input_ids, imgs, seq_mask, spatial = _uocr_build_inputs(
1085
+ tokenizer, prompt, images, crop_mode=crop_mode, base_size=base_size,
1086
+ image_size=image_size, max_crops=max_crops)
1087
+ res = self._uocr_generate(
1088
+ tokenizer, input_ids, imgs, seq_mask, spatial,
1089
+ max_new_tokens=(max_new_tokens if max_new_tokens else max(1, max_length - input_ids.numel())),
1090
+ max_length=max_length, temperature=temperature, page_count_guard=page_count_guard,
1091
+ max_page_tokens=max_page_tokens, ngram_size=no_repeat_ngram_size,
1092
+ ngram_window=(ngram_window if ngram_window > 0 else 100),
1093
+ ngram_whitelist=set(ngram_whitelist_token_ids or []), dash_topm=dash_topm,
1094
+ stream=not eval_mode, tps_interval=tps_interval)
1095
+ outputs = res['text']
1096
+ if test_compress and n_img:
1097
+ out_len = len(tokenizer.encode(outputs, add_special_tokens=False))
1098
+ n_vis = int(seq_mask.sum())
1099
+ print('=' * 50)
1100
+ print('image tokens: ', n_vis, ' output tokens: ', out_len,
1101
+ ' compression ratio: ', round(out_len / max(1, n_vis), 2))
1102
+ print('=' * 50)
1103
+ if save_results and output_path:
1104
+ self._uocr_save_results(outputs, images, output_path)
1105
+ if return_dict:
1106
+ res['prompt'] = prompt
1107
+ return res
1108
+ return outputs
1109
+
1110
+ def infer_multi(self, tokenizer, prompt='', image_files=None, output_path='', image_size=640,
1111
+ save_results=False, max_length=131072, tps_interval=0, no_repeat_ngram_size=0,
1112
+ ngram_window=0, temperature=0.0, **kwargs):
1113
+ """Backward-compatible multi-page entry point -> ``infer`` (base mode, no crop).
1114
+ Returns (outputs, output_tokens) like before."""
1115
+ if not image_files:
1116
+ raise ValueError('image_files must be a non-empty list for multi-image inference!')
1117
+ kwargs.setdefault('crop_mode', False)
1118
+ res = self.infer(tokenizer, prompt=prompt.replace('<image>', '') if prompt.count('<image>') == 1 else prompt,
1119
+ image_file=list(image_files), output_path=output_path, image_size=image_size,
1120
+ save_results=save_results, max_length=max_length, tps_interval=tps_interval,
1121
+ no_repeat_ngram_size=no_repeat_ngram_size, ngram_window=ngram_window,
1122
+ temperature=temperature, return_dict=True, **kwargs)
1123
+ return res['text'], len(res['token_ids'])
1124
+
1125
+ @torch.no_grad()
1126
+ def _uocr_generate(self, tokenizer, input_ids, imgs, seq_mask, spatial, max_new_tokens=16384,
1127
+ max_length=131072, temperature=0.0, page_count_guard=True, max_page_tokens=20000,
1128
+ ngram_size=0, ngram_window=100, ngram_whitelist=None, dash_topm=16,
1129
+ stream=False, tps_interval=0):
1130
+ """Prefill + token-by-token decode on a DynamicCache (no HF generate()).
1131
+ Only the last position goes through lm_head (full-sequence logits at 30k tokens x
1132
+ 129k vocab would not fit)."""
1133
+ import time
1134
+ from transformers.cache_utils import DynamicCache
1135
+ device = self.lm_head.weight.device
1136
+ ids = input_ids.unsqueeze(0).to(device)
1137
+ prompt_ids = input_ids.tolist()
1138
+ page_id = int((self.config.sparse_attention_config or {}).get('page_token_id', 128827)) \
1139
+ if isinstance(getattr(self.config, 'sparse_attention_config', None), dict) else 128827
1140
+ eos_id = int(tokenizer.eos_token_id)
1141
+ expected_pages = prompt_ids.count(page_id) if page_count_guard else 0
1142
+ guard_st = {'marker_cum': 0, 'scanned': 0, 'page_len': 0}
1143
+ kw = {}
1144
+ if imgs is not None:
1145
+ kw = dict(images=[(imgs[0].to(device), imgs[1].to(device), spatial)],
1146
+ images_seq_mask=seq_mask.unsqueeze(0).to(device),
1147
+ images_spatial_crop=spatial)
1148
+
1149
+ # decode-time knobs: no ring-buffer SWA (vLLM attends via DASH), vLLM-parity router
1150
+ cfg = self.config
1151
+ saved_cfg = (getattr(cfg, '_ring_window', None), getattr(cfg, 'sliding_window', None))
1152
+ cfg._ring_window = None
1153
+ cfg.sliding_window = None
1154
+ dash_state = configure_dash_inference(self.model, True, topm=dash_topm)
1155
+ streamer = TextStreamer(tokenizer, skip_prompt=False, skip_special_tokens=False) if stream else None
1156
+ gen, finish = [], 'length'
1157
+ t0 = time.time()
1158
+ t_first = None
1159
+ try:
1160
+ cache = DynamicCache()
1161
+ S = ids.shape[1]
1162
+ out = self.model(input_ids=ids, past_key_values=cache, use_cache=True,
1163
+ position_ids=torch.arange(S, device=device)[None], return_dict=True, **kw)
1164
+ cache = out.past_key_values
1165
+ h = out.last_hidden_state[:, -1:]
1166
+ for step in range(max_new_tokens):
1167
+ logits = self.lm_head(h).float()[0, -1]
1168
+ if ngram_size > 0:
1169
+ logits = _uocr_no_repeat_ngram(logits, gen, ngram_size, ngram_window, ngram_whitelist or set())
1170
+ if expected_pages > 0:
1171
+ logits = _uocr_page_count_guard(logits, gen, expected_pages, page_id, eos_id,
1172
+ max_page_tokens, guard_st)
1173
+ if temperature and temperature > 0:
1174
+ nxt = int(torch.multinomial(torch.softmax(logits / temperature, dim=-1), 1))
1175
+ else:
1176
+ nxt = int(torch.argmax(logits))
1177
+ if t_first is None:
1178
+ t_first = time.time()
1179
+ if nxt == eos_id:
1180
+ finish = 'stop'
1181
+ break
1182
+ gen.append(nxt)
1183
+ if streamer is not None:
1184
+ streamer.put(torch.tensor([nxt]))
1185
+ if tps_interval and len(gen) % tps_interval == 0:
1186
+ dt = time.time() - t_first
1187
+ print(f'\n[TPS] tokens={len(gen)} avg={len(gen) / max(dt, 1e-6):.1f} t/s', flush=True)
1188
+ pos = S + len(gen) - 1
1189
+ if pos + 1 >= max_length:
1190
+ break
1191
+ out = self.model(input_ids=torch.tensor([[nxt]], device=device), past_key_values=cache,
1192
+ use_cache=True, position_ids=torch.tensor([[pos]], device=device),
1193
+ return_dict=True)
1194
+ cache = out.past_key_values
1195
+ h = out.last_hidden_state
1196
+ finally:
1197
+ configure_dash_inference(self.model, False, state=dash_state)
1198
+ cfg._ring_window, cfg.sliding_window = saved_cfg
1199
+ if streamer is not None:
1200
+ streamer.end()
1201
+ text = tokenizer.decode(gen, skip_special_tokens=False)
1202
+ return {'text': text, 'token_ids': gen, 'prompt_token_ids': prompt_ids,
1203
+ 'finish_reason': finish, 'num_pages_out': gen.count(page_id),
1204
+ 'expected_pages': expected_pages, 'time': time.time() - t0,
1205
+ 'prefill_time': (t_first - t0) if t_first else None}
1206
+
1207
+ def _uocr_save_results(self, outputs, images, output_path):
1208
+ os.makedirs(output_path, exist_ok=True)
1209
+ os.makedirs(f'{output_path}/images', exist_ok=True)
1210
+ pages = outputs.split('<PAGE>')[1:] if '<PAGE>' in outputs else [outputs]
1211
+ processed = []
1212
+ for page_idx, page_output in enumerate(pages):
1213
+ page_output = page_output.strip()
1214
+ if page_idx < len(images):
1215
+ matches_ref, matches_images, mathes_other = re_match(page_output)
1216
+ prefix = f'page_{page_idx}_'
1217
+ result = process_image_with_refs(images[page_idx].copy(), matches_ref, output_path,
1218
+ image_prefix=prefix)
1219
+ result.save(f'{output_path}/result_with_boxes_{page_idx}.jpg')
1220
+ for idx, m in enumerate(matches_images):
1221
+ page_output = page_output.replace(m, f'![](images/{prefix}{idx}.jpg)\n')
1222
+ for m in mathes_other:
1223
+ page_output = page_output.replace(m, '').replace('\\coloneqq', ':=').replace('\\eqqcolon', '=:')
1224
+ processed.append(page_output)
1225
+ with open(f'{output_path}/result.md', 'w', encoding='utf-8') as f:
1226
+ f.write('<PAGE>\n' + '\n<PAGE>\n'.join(processed))
1227
+ with open(f'{output_path}/result_raw.md', 'w', encoding='utf-8') as f:
1228
+ f.write(outputs)
packing_patch.py ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Sequence-packing support for the sparse-attention path.
2
+
3
+ Ported from Megatron-Bridge (``deepseek_ocr2/modeling_deepseek_ocr_mcore.py``):
4
+ the two-stage warmup/sparse pipeline there runs with packing, and does so by
5
+ making every causal / local-window / block mask *segment aware* instead of
6
+ assuming a single contiguous sequence per row.
7
+
8
+ Why this is needed
9
+ -------------------
10
+ With packing, ms-swift concatenates several samples into one row and emits
11
+ per-sample-RESET ``position_ids`` (``[0,1,..,L1-1, 0,1,..,L2-1, ...]``; see
12
+ ``swift/template/base.py::packing_row``). The sparse code compares the query
13
+ ``position_ids`` against ``k_positions = arange(k_len)`` (the GLOBAL flat key
14
+ index) to build causal / local masks. With reset positions that comparison is
15
+ wrong (a sample-2 query at local pos 0 would be "allowed" to see only key 0,
16
+ which belongs to sample 1), so packing silently corrupts attention.
17
+
18
+ Fix (mirrors Megatron-Bridge)
19
+ -----------------------------
20
+ * Derive a per-token SEGMENT id from the ``position_ids`` resets.
21
+ * Evaluate causal / local windows in GLOBAL key space using the flat query
22
+ index (for a packed row with batch==1 the flat index *is* the global
23
+ position, and ``k_positions`` is already global).
24
+ * AND every keep-mask with a ``same_segment`` constraint so no query attends
25
+ across a sample boundary.
26
+ * Bypass the fused Triton kernels when packed (they assume ``position_ids ==
27
+ arange`` and a single segment); the eager PyTorch paths below are made
28
+ segment aware instead.
29
+
30
+ batch==1 is assumed for packed rows (``padding_free``/packing collapses the
31
+ batch dimension into the sequence).
32
+
33
+ Block-grid alignment (sparse stage)
34
+ -----------------------------------
35
+ Key blocks are cut on the FLAT (global) token index, so a sample that does not
36
+ start on a ``block_size`` boundary gets a block grid shifted by
37
+ ``start % block_size`` relative to single-sample inference. That is safe (the
38
+ segment masks above still prevent any cross-sample attention) but it is a
39
+ train/inference discrepancy in *how the keys are grouped*, plus the first block
40
+ of each sample straddles the previous one and gets trimmed. Run packing with
41
+ ``PACKING_ALIGN_SIZE=<block_size>`` (ms-swift ``Template._align_packing_row``)
42
+ to make every sample start on a block boundary; ``check_block_alignment`` warns
43
+ once when it is not the case.
44
+ """
45
+ import torch
46
+
47
+
48
+ def is_packed(position_ids):
49
+ """A packed row restarts ``position_ids`` at 0 at each sample boundary, so a
50
+ 0 appears at some column >= 1. A plain (non-packed) sequence only has its
51
+ single 0 at column 0. Robust for training (no left padding under
52
+ padding_free)."""
53
+ if position_ids is None or position_ids.dim() != 2 or position_ids.shape[1] < 2:
54
+ return False
55
+ return bool((position_ids[:, 1:] == 0).any())
56
+
57
+
58
+ def segment_ids(position_ids):
59
+ """[B, S] long: 0-based segment index of every token, from position_ids
60
+ resets. ``(position_ids == 0)`` marks each segment start; cumsum-1 numbers
61
+ them. For a non-packed row this is all zeros (single segment)."""
62
+ return (position_ids == 0).to(torch.long).cumsum(dim=1) - 1
63
+
64
+
65
+ def packed_masks(position_ids, k_positions, q_index, local_window):
66
+ """All masks are built in GLOBAL key space and are [B, 1, C, Sk] bool.
67
+
68
+ Args:
69
+ position_ids: [B, S] reset positions (training prefill: S == k_len).
70
+ k_positions: [Sk] global key indices == ``arange(k_len)``.
71
+ q_index: [C] global (flat) indices of the query rows in this
72
+ chunk (e.g. ``arange(c0, c1)`` or the strided
73
+ ``sel`` used by the KL loss). These ARE the global
74
+ query positions for a packed batch==1 row.
75
+ local_window: int inclusive local window length in tokens.
76
+
77
+ Returns dict with:
78
+ token_future [B,1,C,Sk] key is strictly after the query (to mask out)
79
+ local_token [B,1,C,Sk] key in [gpos-local_window+1, gpos]
80
+ same_seg [B,1,C,Sk] key and query share a segment
81
+ gpos [B,C] global query positions
82
+ """
83
+ device = position_ids.device
84
+ B = position_ids.shape[0]
85
+ Sk = k_positions.shape[0]
86
+ seg = segment_ids(position_ids) # [B, S]
87
+ seg_k = seg[:, :Sk] # [B, Sk]
88
+ q_index = q_index.to(device=device, dtype=torch.long)
89
+ seg_q = seg.index_select(1, q_index) # [B, C]
90
+ gpos = q_index.view(1, -1).expand(B, -1) # [B, C] global q pos
91
+
92
+ kp = k_positions.view(1, 1, 1, -1) # [1,1,1,Sk]
93
+ gpos_e = gpos[:, None, :, None] # [B,1,C,1]
94
+ token_future = kp > gpos_e
95
+ ls = (gpos - (local_window - 1)).clamp(min=0)[:, None, :, None]
96
+ local_token = (kp >= ls) & (kp <= gpos_e)
97
+ same_seg = seg_k[:, None, None, :] == seg_q[:, None, :, None]
98
+ return {
99
+ 'token_future': token_future,
100
+ 'local_token': local_token,
101
+ 'same_seg': same_seg,
102
+ 'gpos': gpos,
103
+ }
104
+
105
+
106
+ def segment_of(position_ids, token_indices):
107
+ """Gather the segment id for arbitrary GLOBAL token indices.
108
+
109
+ Args:
110
+ position_ids: [B, S]
111
+ token_indices: [B, ...] long, global key indices (clamped by caller).
112
+ Returns tensor shaped like ``token_indices`` with the segment id of each.
113
+ """
114
+ seg = segment_ids(position_ids) # [B, S]
115
+ flat = token_indices.reshape(token_indices.shape[0], -1)
116
+ out = seg.gather(1, flat.clamp(min=0, max=seg.shape[1] - 1))
117
+ return out.reshape(token_indices.shape)
118
+
119
+
120
+ def segment_starts(position_ids):
121
+ """[N] long: GLOBAL start index of every segment in a packed row (batch 0).
122
+ ``start = flat_index - position_ids`` at the reset points."""
123
+ return (position_ids[0] == 0).nonzero(as_tuple=True)[0]
124
+
125
+
126
+ _ALIGN_WARNED = False
127
+
128
+
129
+ def check_block_alignment(position_ids, block_size):
130
+ """one-shot warning when a packed row's samples do NOT start on a
131
+ ``block_size`` boundary.
132
+
133
+ The sparse stage cuts its key blocks on the FLAT (global) token index, so a sample
134
+ starting at ``s`` gets a block grid shifted by ``s % block_size`` relative to
135
+ single-sample inference (which starts at 0), and its first block straddles the previous
136
+ sample (the segment mask then trims it, wasting part of the remote top-k budget).
137
+ Correctness is unaffected (no cross-sample leakage), but the train/inference block
138
+ partitioning differs. Set ``PACKING_ALIGN_SIZE=<block_size>`` so ms-swift's
139
+ ``packing_row`` pads every sample up to that multiple and the two grids coincide.
140
+ """
141
+ global _ALIGN_WARNED
142
+ if _ALIGN_WARNED or position_ids is None or block_size <= 1:
143
+ return
144
+ starts = segment_starts(position_ids)
145
+ n_bad = int((starts % block_size != 0).sum())
146
+ if n_bad:
147
+ _ALIGN_WARNED = True
148
+ print(f'[PackingAlign] {n_bad}/{starts.numel()} packed samples do not start on a '
149
+ f'{block_size}-token block boundary -> block grid is shifted w.r.t. '
150
+ f'single-sample inference. Set PACKING_ALIGN_SIZE={block_size} to align.',
151
+ flush=True)
152
+
153
+
154
+ def dense_segment_keep(position_ids, device):
155
+ """Boolean [B, 1, S, S] keep-mask for the DENSE (warmup) main branch:
156
+ True where a query may attend (same segment AND causal). Passed straight to
157
+ ``scaled_dot_product_attention(attn_mask=...)`` (bool: True == attend).
158
+ Mirrors the ``same_seg & causal`` SDPA mask of Megatron-Bridge's dense path.
159
+
160
+ NOTE: this materialises a [B, 1, S, S] bool tensor (~1 GiB at S=32k, B=1).
161
+ A chunked-query SDPA (as in MB's ``modeling_deepseek_ocr_mcore`` cp>1 loop)
162
+ would avoid it and is a follow-up memory optimisation."""
163
+ B, S = position_ids.shape
164
+ seg = segment_ids(position_ids) # [B, S]
165
+ pos = torch.arange(S, device=device) # global positions
166
+ causal = pos[None, :] <= pos[:, None] # [S, S] q >= k
167
+ same_seg = seg[:, :, None] == seg[:, None, :] # [B, S, S]
168
+ return (same_seg & causal[None])[:, None] # [B, 1, S, S] bool
pyproject.toml ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [build-system]
2
+ requires = ["setuptools>=61"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "unlimited-ocr-config-runtime"
7
+ version = "0.1.0"
8
+ requires-python = ">=3.10"
9
+
10
+ [tool.setuptools.packages.find]
11
+ include = ["unlimited_ocr_runtime*"]
sparse_gather_e2e_triton.py ADDED
@@ -0,0 +1,407 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """fused Triton gather-attention WITH a differentiable per-block bias.
2
+
3
+ This is the high-performance backend for HiLS-style end-to-end selector training.
4
+ It is a copy of ``sparse_gather_triton.py`` (the production sparse-stage kernel, whose
5
+ union/causal/local-window semantics are already battle-tested) with ONE addition:
6
+
7
+ * a per-(token, kv-block) additive bias ``blk_bias[B, T, M]`` (M = num kv blocks) is
8
+ added to the logits of the tokens attended through the SELECTED (remote) blocks --
9
+ exactly HiLS's inter-chunk gate. The forward adds it; the backward returns
10
+ ``d_blk_bias`` so the LM-loss gradient flows back through the bias into whatever
11
+ produced it (the indexer q/k projections). Local-window keys get zero bias, matching
12
+ the eager reference in ``DeepseekV2Indexer.e2e_sparse_attention``.
13
+
14
+ The differentiable log-sum-exp block mass itself is still computed in PyTorch (validated
15
+ by test_e2e_math.py); this kernel only replaces the heavy gather + softmax + matmul and
16
+ their backward, and threads the bias gradient through.
17
+
18
+ Semantics (must match the eager e2e reference):
19
+ logit(query t, key j in selected block b) = scale * q_t . k_j + blk_bias[t, b]
20
+ for j causal and j < local_start(t)
21
+ logit(query t, key j in local window) = scale * q_t . k_j (bias 0)
22
+ Single index head (block indices shared across kv heads), single contiguous causal
23
+ sequence, q_len == k_len (training prefill). The wrapper returns None on unsupported
24
+ shapes so the caller falls back to the eager torch path.
25
+ """
26
+
27
+ import torch
28
+ import triton
29
+ import triton.language as tl
30
+
31
+ # Force transformers' dynamic-module loader to copy this file (see the twin sentinel
32
+ # in sparse_gather_triton.py). modeling_unlimitedocr.py imports this name at top level.
33
+ SPARSE_E2E_SENTINEL = 1
34
+
35
+
36
+ @triton.jit(do_not_specialize=['T'])
37
+ def _e2e_attn_fwd_kernel(
38
+ q, k, v, o, lse, scale, block_indices, blk_bias,
39
+ T,
40
+ H: tl.constexpr, HQ: tl.constexpr, G: tl.constexpr, GP: tl.constexpr,
41
+ K: tl.constexpr, V: tl.constexpr, S: tl.constexpr, M: tl.constexpr,
42
+ BS: tl.constexpr, BK: tl.constexpr, BV: tl.constexpr,
43
+ LW: tl.constexpr, NLB: tl.constexpr,
44
+ ):
45
+ i_t, i_v, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
46
+ i_b, i_h = i_bh // H, i_bh % H
47
+ bos = i_b * T
48
+
49
+ k += (bos * H + i_h) * K
50
+ v += (bos * H + i_h) * V
51
+ block_indices += (bos + i_t) * S
52
+ blk_bias += (bos + i_t) * M
53
+
54
+ o_g = tl.arange(0, GP)
55
+ o_k = tl.arange(0, BK)
56
+ o_v = tl.arange(0, BV)
57
+ m_g = o_g < G
58
+ m_k = o_k < K
59
+ m_v = (i_v * BV + o_v) < V
60
+
61
+ p_q = q + (bos + i_t) * HQ * K + (i_h * G + o_g)[:, None] * K + o_k[None, :]
62
+ b_q = tl.load(p_q, mask=m_g[:, None] & m_k[None, :], other=0.)
63
+ b_q = (b_q * scale).to(b_q.dtype)
64
+
65
+ b_o = tl.zeros([GP, BV], dtype=tl.float32)
66
+ b_m = tl.full([GP], float('-inf'), dtype=tl.float32)
67
+ b_acc = tl.zeros([GP], dtype=tl.float32)
68
+ ls = i_t - LW + 1
69
+
70
+ # --- remote: selected non-local blocks (keys < local_start), WITH per-block bias ---
71
+ for i in range(S):
72
+ bid = tl.load(block_indices + i).to(tl.int32)
73
+ i_s = bid * BS
74
+ if i_s <= i_t and i_s >= 0:
75
+ b_bias = tl.load(blk_bias + bid).to(tl.float32)
76
+ p_k = tl.make_block_ptr(k, (K, T), (1, H * K), (0, i_s), (BK, BS), (0, 1))
77
+ p_v = tl.make_block_ptr(v, (T, V), (H * V, 1), (i_s, i_v * BV), (BS, BV), (1, 0))
78
+ b_k = tl.load(p_k, boundary_check=(0, 1))
79
+ b_v = tl.load(p_v, boundary_check=(0, 1))
80
+ b_s = tl.dot(b_q, b_k)
81
+ key = i_s + tl.arange(0, BS)
82
+ b_s = tl.where(((i_t >= key) & (key < ls))[None, :], b_s + b_bias, float('-inf'))
83
+ # safe online softmax: a fully-masked tile keeps b_m at -inf without NaN
84
+ # (b_m_safe==0 there, so alpha=exp(-inf)=0 and b_p=exp(-inf)=0).
85
+ b_m_new = tl.maximum(b_m, tl.max(b_s, 1))
86
+ b_m_safe = tl.where(b_m_new == float('-inf'), 0.0, b_m_new)
87
+ b_r = tl.exp(b_m - b_m_safe)
88
+ b_p = tl.exp(b_s - b_m_safe[:, None])
89
+ b_acc = b_acc * b_r + tl.sum(b_p, 1)
90
+ b_o = b_o * b_r[:, None] + tl.dot(b_p.to(b_q.dtype), b_v)
91
+ b_m = b_m_new
92
+
93
+ # --- local window blocks (keys >= local_start), NO bias ---
94
+ q_blk = i_t // BS
95
+ for lb in range(NLB):
96
+ i_s = (q_blk - lb) * BS
97
+ if i_s >= 0:
98
+ p_k = tl.make_block_ptr(k, (K, T), (1, H * K), (0, i_s), (BK, BS), (0, 1))
99
+ p_v = tl.make_block_ptr(v, (T, V), (H * V, 1), (i_s, i_v * BV), (BS, BV), (1, 0))
100
+ b_k = tl.load(p_k, boundary_check=(0, 1))
101
+ b_v = tl.load(p_v, boundary_check=(0, 1))
102
+ b_s = tl.dot(b_q, b_k)
103
+ key = i_s + tl.arange(0, BS)
104
+ b_s = tl.where(((i_t >= key) & (key >= ls))[None, :], b_s, float('-inf'))
105
+ b_m_new = tl.maximum(b_m, tl.max(b_s, 1))
106
+ b_m_safe = tl.where(b_m_new == float('-inf'), 0.0, b_m_new)
107
+ b_r = tl.exp(b_m - b_m_safe)
108
+ b_p = tl.exp(b_s - b_m_safe[:, None])
109
+ b_acc = b_acc * b_r + tl.sum(b_p, 1)
110
+ b_o = b_o * b_r[:, None] + tl.dot(b_p.to(b_q.dtype), b_v)
111
+ b_m = b_m_new
112
+
113
+ b_safe = tl.where(b_acc > 0, b_acc, 1.0)
114
+ b_o = b_o / b_safe[:, None]
115
+ b_lse = b_m + tl.log(b_safe)
116
+
117
+ p_o = o + (bos + i_t) * HQ * V + (i_h * G + o_g)[:, None] * V + (i_v * BV + o_v)[None, :]
118
+ tl.store(p_o, b_o.to(q.dtype.element_ty), mask=m_g[:, None] & m_v[None, :])
119
+ if i_v == 0:
120
+ p_lse = lse + (bos + i_t) * HQ + i_h * G + o_g
121
+ tl.store(p_lse, b_lse, mask=m_g)
122
+
123
+
124
+ @triton.jit(do_not_specialize=['T'])
125
+ def _e2e_attn_bwd_dq_kernel(
126
+ q, k, v, lse, delta, do, dq, scale, block_indices, blk_bias, d_bias,
127
+ T,
128
+ H: tl.constexpr, HQ: tl.constexpr, G: tl.constexpr, GP: tl.constexpr,
129
+ K: tl.constexpr, V: tl.constexpr, S: tl.constexpr, M: tl.constexpr,
130
+ BS: tl.constexpr, BK: tl.constexpr, BV: tl.constexpr,
131
+ LW: tl.constexpr, NLB: tl.constexpr,
132
+ ):
133
+ i_t, i_v, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
134
+ i_b, i_h = i_bh // H, i_bh % H
135
+ bos = i_b * T
136
+
137
+ k += (bos * H + i_h) * K
138
+ v += (bos * H + i_h) * V
139
+ block_indices += (bos + i_t) * S
140
+ blk_bias += (bos + i_t) * M
141
+ d_bias += (bos + i_t) * M
142
+
143
+ o_g = tl.arange(0, GP)
144
+ o_k = tl.arange(0, BK)
145
+ o_v = tl.arange(0, BV)
146
+ m_g = o_g < G
147
+ m_k = o_k < K
148
+ m_v = (i_v * BV + o_v) < V
149
+
150
+ p_q = q + (bos + i_t) * HQ * K + (i_h * G + o_g)[:, None] * K + o_k[None, :]
151
+ b_q = tl.load(p_q, mask=m_g[:, None] & m_k[None, :], other=0.)
152
+ b_q = (b_q * scale).to(b_q.dtype)
153
+
154
+ p_do = do + (bos + i_t) * HQ * V + (i_h * G + o_g)[:, None] * V + (i_v * BV + o_v)[None, :]
155
+ b_do = tl.load(p_do, mask=m_g[:, None] & m_v[None, :], other=0.)
156
+ p_lse = lse + (bos + i_t) * HQ + i_h * G + o_g
157
+ b_lse = tl.load(p_lse, mask=m_g, other=float('inf'))
158
+ p_delta = delta + (bos + i_t) * HQ + i_h * G + o_g
159
+ b_delta = tl.load(p_delta, mask=m_g, other=0.)
160
+
161
+ b_dq = tl.zeros([GP, BK], dtype=tl.float32)
162
+ ls = i_t - LW + 1
163
+
164
+ # --- remote: selected non-local blocks (bias added; accumulate d_bias) ---
165
+ for i in range(S):
166
+ bid = tl.load(block_indices + i).to(tl.int32)
167
+ i_s = bid * BS
168
+ if i_s <= i_t and i_s >= 0:
169
+ b_bias = tl.load(blk_bias + bid).to(tl.float32)
170
+ p_k = tl.make_block_ptr(k, (K, T), (1, H * K), (0, i_s), (BK, BS), (0, 1))
171
+ p_v = tl.make_block_ptr(v, (V, T), (1, H * V), (i_v * BV, i_s), (BV, BS), (0, 1))
172
+ b_k = tl.load(p_k, boundary_check=(0, 1))
173
+ b_v = tl.load(p_v, boundary_check=(0, 1))
174
+ b_s = tl.dot(b_q, b_k)
175
+ key = i_s + tl.arange(0, BS)
176
+ b_p = tl.exp((b_s + b_bias) - b_lse[:, None])
177
+ b_p = tl.where(((i_t >= key) & (key < ls))[None, :], b_p, 0.)
178
+ b_dp = tl.dot(b_do, b_v)
179
+ b_ds = b_p * (b_dp.to(tl.float32) - b_delta[:, None])
180
+ b_dq += tl.dot(b_ds.to(b_k.dtype), tl.trans(b_k))
181
+ # d_bias[t, bid] += sum over this kv-head-group's query rows & keys of ds.
182
+ # Atomics accumulate the remaining kv heads (other programs); padded head
183
+ # rows have lse=+inf -> b_p=0, non-kept keys -> b_p=0, so they add nothing.
184
+ if i_v == 0:
185
+ tl.atomic_add(d_bias + bid, tl.sum(b_ds))
186
+
187
+ # --- local window blocks (no bias, no d_bias) ---
188
+ q_blk = i_t // BS
189
+ for lb in range(NLB):
190
+ i_s = (q_blk - lb) * BS
191
+ if i_s >= 0:
192
+ p_k = tl.make_block_ptr(k, (K, T), (1, H * K), (0, i_s), (BK, BS), (0, 1))
193
+ p_v = tl.make_block_ptr(v, (V, T), (1, H * V), (i_v * BV, i_s), (BV, BS), (0, 1))
194
+ b_k = tl.load(p_k, boundary_check=(0, 1))
195
+ b_v = tl.load(p_v, boundary_check=(0, 1))
196
+ b_s = tl.dot(b_q, b_k)
197
+ key = i_s + tl.arange(0, BS)
198
+ b_p = tl.exp(b_s - b_lse[:, None])
199
+ b_p = tl.where(((i_t >= key) & (key >= ls))[None, :], b_p, 0.)
200
+ b_dp = tl.dot(b_do, b_v)
201
+ b_ds = b_p * (b_dp.to(tl.float32) - b_delta[:, None])
202
+ b_dq += tl.dot(b_ds.to(b_k.dtype), tl.trans(b_k))
203
+ b_dq *= scale
204
+
205
+ p_dq = dq + (bos + i_t) * HQ * K + (i_h * G + o_g)[:, None] * K + o_k[None, :]
206
+ tl.store(p_dq, b_dq.to(q.dtype.element_ty), mask=m_g[:, None] & m_k[None, :])
207
+
208
+
209
+ @triton.jit(do_not_specialize=['T'])
210
+ def _e2e_attn_bwd_dkv_kernel(
211
+ q, k, v, lse, delta, do, dk, dv, block_mask, blk_bias, scale,
212
+ T,
213
+ H: tl.constexpr, HQ: tl.constexpr, G: tl.constexpr, GP: tl.constexpr,
214
+ K: tl.constexpr, V: tl.constexpr, M: tl.constexpr,
215
+ BS: tl.constexpr, BK: tl.constexpr, BV: tl.constexpr,
216
+ LW: tl.constexpr,
217
+ ):
218
+ i_v, i_s, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
219
+ i_b, i_h = i_bh // H, i_bh % H
220
+ bos = i_b * T
221
+
222
+ p_k = tl.make_block_ptr(k + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_s * BS, 0), (BS, BK), (1, 0))
223
+ p_v = tl.make_block_ptr(v + (bos * H + i_h) * V, (T, V), (H * V, 1), (i_s * BS, i_v * BV), (BS, BV), (1, 0))
224
+ p_dk = tl.make_block_ptr(dk + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_s * BS, 0), (BS, BK), (1, 0))
225
+ p_dv = tl.make_block_ptr(dv + (bos * H + i_h) * V, (T, V), (H * V, 1), (i_s * BS, i_v * BV), (BS, BV), (1, 0))
226
+
227
+ o_g = tl.arange(0, GP)
228
+ o_k = tl.arange(0, BK)
229
+ o_v = tl.arange(0, BV)
230
+ m_g = o_g < G
231
+ m_k = o_k < K
232
+ m_v = (i_v * BV + o_v) < V
233
+
234
+ b_k = tl.load(p_k, boundary_check=(0, 1))
235
+ b_dk = tl.zeros([BS, BK], dtype=tl.float32)
236
+ b_v = tl.load(p_v, boundary_check=(0, 1))
237
+ b_dv = tl.zeros([BS, BV], dtype=tl.float32)
238
+
239
+ key = i_s * BS + tl.arange(0, BS) # [BS] key positions of this kv block
240
+
241
+ for i in range(i_s * BS, T):
242
+ b_flag = tl.load(block_mask + (bos + i) * M + i_s)
243
+ _proc = (b_flag != 0) | (i < (i_s + 1) * BS + LW)
244
+ if _proc:
245
+ p_q = q + (bos + i) * HQ * K + (i_h * G + o_g)[:, None] * K + o_k[None, :]
246
+ b_q = tl.load(p_q, mask=m_g[:, None] & m_k[None, :], other=0.)
247
+ b_q = (b_q * scale).to(b_q.dtype)
248
+
249
+ p_do = do + (bos + i) * HQ * V + (i_h * G + o_g)[:, None] * V + (i_v * BV + o_v)[None, :]
250
+ b_do = tl.load(p_do, mask=m_g[:, None] & m_v[None, :], other=0.)
251
+ p_lse = lse + (bos + i) * HQ + i_h * G + o_g
252
+ b_lse = tl.load(p_lse, mask=m_g, other=float('inf'))
253
+ p_delta = delta + (bos + i) * HQ + i_h * G + o_g
254
+ b_delta = tl.load(p_delta, mask=m_g, other=0.)
255
+
256
+ ls = i - LW + 1
257
+ is_local = key > (i - LW) # [BS]
258
+ is_remote = key < ls # [BS]
259
+ # bias applies only to REMOTE keys of a SELECTED block (dense cell is 0 for
260
+ # non-selected blocks) -> matches the fwd remote-loop bias, local keys get 0.
261
+ b_bias = tl.load(blk_bias + (bos + i) * M + i_s).to(tl.float32)
262
+ bias_key = tl.where(is_remote, b_bias, 0.0) # [BS]
263
+
264
+ b_s = tl.dot(b_k, tl.trans(b_q)) # [BS, GP]
265
+ b_p = tl.exp((b_s + bias_key[:, None]) - b_lse[None, :])
266
+ causal = i >= key
267
+ keep = causal & (is_local | (b_flag != 0))
268
+ b_p = tl.where(keep[:, None], b_p, 0.)
269
+ b_dv += tl.dot(b_p.to(b_do.dtype), b_do)
270
+ b_dp = tl.dot(b_v, tl.trans(b_do))
271
+ b_ds = b_p * (b_dp.to(tl.float32) - b_delta[None, :])
272
+ b_dk += tl.dot(b_ds.to(b_q.dtype), b_q)
273
+
274
+ tl.store(p_dk, b_dk.to(q.dtype.element_ty), boundary_check=(0, 1))
275
+ tl.store(p_dv, b_dv.to(q.dtype.element_ty), boundary_check=(0, 1))
276
+
277
+
278
+ def _meta(q, k, v, sel, block_size):
279
+ B, T, HQ, K = q.shape
280
+ H = k.shape[2]
281
+ V = v.shape[3]
282
+ S = sel.shape[-1]
283
+ G = HQ // H
284
+ GP = max(16, triton.next_power_of_2(G))
285
+ BK = triton.next_power_of_2(K)
286
+ BV = min(128, triton.next_power_of_2(V))
287
+ if BV < V:
288
+ raise NotImplementedError(f"head_dim V={V} > 128 not supported")
289
+ if BK > 256:
290
+ raise NotImplementedError(f"head_dim K={K} > 256 not supported")
291
+ return B, T, HQ, H, K, V, S, G, GP, block_size, BK, BV
292
+
293
+
294
+ def _local_meta(block_size, local_window):
295
+ lw = block_size if local_window is None else int(local_window)
296
+ lw = max(1, lw)
297
+ nlb = (lw + block_size - 1) // block_size + 1
298
+ return lw, nlb
299
+
300
+
301
+ class _E2EGatherAttnFn(torch.autograd.Function):
302
+ @staticmethod
303
+ def forward(ctx, q, k, v, sel, blk_bias, block_size, scale, local_window):
304
+ B, T, HQ, H, K, V, S, G, GP, BS, BK, BV = _meta(q, k, v, sel, block_size)
305
+ LW, NLB = _local_meta(BS, local_window)
306
+ M = triton.cdiv(T, BS)
307
+ assert blk_bias.shape[-1] == M, f"blk_bias last dim {blk_bias.shape[-1]} != M {M}"
308
+ blk_bias = blk_bias.contiguous().float()
309
+
310
+ o = torch.empty(B, T, HQ, V, dtype=q.dtype, device=q.device)
311
+ lse = torch.empty(B, T, HQ, dtype=torch.float32, device=q.device)
312
+ _e2e_attn_fwd_kernel[(T, 1, B * H)](
313
+ q, k, v, o, lse, scale, sel, blk_bias,
314
+ T=T, H=H, HQ=HQ, G=G, GP=GP, K=K, V=V, S=S, M=M, BS=BS, BK=BK, BV=BV,
315
+ LW=LW, NLB=NLB, num_warps=4,
316
+ )
317
+ ctx.save_for_backward(q, k, v, sel, blk_bias, o, lse)
318
+ ctx.block_size = block_size
319
+ ctx.scale = scale
320
+ ctx.local_window = local_window
321
+ return o
322
+
323
+ @staticmethod
324
+ def backward(ctx, do):
325
+ q, k, v, sel, blk_bias, o, lse = ctx.saved_tensors
326
+ block_size, scale, local_window = ctx.block_size, ctx.scale, ctx.local_window
327
+ B, T, HQ, H, K, V, S, G, GP, BS, BK, BV = _meta(q, k, v, sel, block_size)
328
+ LW, NLB = _local_meta(BS, local_window)
329
+ M = triton.cdiv(T, BS)
330
+ do = do.contiguous()
331
+ delta = (o.float() * do.float()).sum(-1) # [B, T, HQ]
332
+
333
+ dq = torch.empty_like(q)
334
+ d_bias = torch.zeros(B, T, M, dtype=torch.float32, device=q.device)
335
+ _e2e_attn_bwd_dq_kernel[(T, 1, B * H)](
336
+ q, k, v, lse, delta, do, dq, scale, sel, blk_bias, d_bias,
337
+ T=T, H=H, HQ=HQ, G=G, GP=GP, K=K, V=V, S=S, M=M, BS=BS, BK=BK, BV=BV,
338
+ LW=LW, NLB=NLB, num_warps=4,
339
+ )
340
+
341
+ block_mask = torch.zeros(B, T, M + 1, dtype=torch.bool, device=q.device)
342
+ block_mask.scatter_(-1, sel.long().clamp(min=0).masked_fill(sel < 0, M), True)
343
+ block_mask = block_mask[..., :M].contiguous()
344
+
345
+ dk = torch.empty_like(k)
346
+ dv = torch.empty_like(v)
347
+ _e2e_attn_bwd_dkv_kernel[(1, M, B * H)](
348
+ q, k, v, lse, delta, do, dk, dv, block_mask, blk_bias, scale,
349
+ T=T, H=H, HQ=HQ, G=G, GP=GP, K=K, V=V, M=M, BS=BS, BK=BK, BV=BV,
350
+ LW=LW, num_warps=4,
351
+ )
352
+ return dq, dk, dv, None, d_bias, None, None, None
353
+
354
+
355
+ def triton_gather_attention_e2e(
356
+ query_states, # [B, HQ, T, D]
357
+ key_states, # [B, Hkv, Sk, D]
358
+ value_states, # [B, Hkv, Sk, D]
359
+ block_indices, # [B, 1, T, S] int, -1 = padding
360
+ blk_bias_slots, # [B, T, S] float, differentiable per-selected-block bias (aligned with block_indices)
361
+ position_ids, # [B, T]
362
+ attention_mask,
363
+ block_size,
364
+ scale,
365
+ local_window=None,
366
+ ):
367
+ """Fused e2e gather-attention entry. Returns [B, HQ, T, D], or None on unsupported
368
+ shape (caller must fall back to the eager torch path). Gradients flow to q/k/v AND
369
+ to blk_bias_slots (-> the indexer projections)."""
370
+ if attention_mask is not None:
371
+ return None
372
+ if block_indices.shape[1] != 1:
373
+ return None
374
+ B, HQ, T, D = query_states.shape
375
+ if key_states.shape[2] != T:
376
+ return None
377
+ if position_ids is None or position_ids.dim() != 2 or position_ids.shape[-1] != T:
378
+ return None
379
+ ar = torch.arange(T, device=position_ids.device)
380
+ if not bool((position_ids == ar).all()):
381
+ return None
382
+
383
+ sel = block_indices[:, 0].contiguous() # [B, T, S]
384
+ bias_slots = blk_bias_slots # [B, T, S] (grad)
385
+ M = triton.cdiv(T, block_size)
386
+
387
+ # dedup repeated block ids within a row (would double-count softmax mass).
388
+ valid = sel >= 0
389
+ sel_sorted, order = sel.sort(dim=-1)
390
+ dup_sorted = torch.zeros_like(valid)
391
+ dup_sorted[..., 1:] = sel_sorted[..., 1:] == sel_sorted[..., :-1]
392
+ dup = torch.zeros_like(valid)
393
+ dup.scatter_(-1, order, dup_sorted)
394
+ sel = sel.masked_fill(dup & valid, -1)
395
+
396
+ # scatter per-slot bias into a dense [B, T, M] tensor keyed by block id (differentiable
397
+ # w.r.t. bias_slots). -1 slots (pad/dedup) route to an overflow column then dropped.
398
+ scatter_idx = sel.long().clamp(min=0).masked_fill(sel < 0, M) # [B, T, S]
399
+ blk_bias_dense = bias_slots.new_zeros(B, T, M + 1).scatter(-1, scatter_idx, bias_slots)
400
+ blk_bias_dense = blk_bias_dense[..., :M].contiguous()
401
+
402
+ sel = sel.to(torch.int32).contiguous()
403
+ q = query_states.transpose(1, 2).contiguous() # [B, T, HQ, D]
404
+ k = key_states.transpose(1, 2).contiguous()
405
+ v = value_states.transpose(1, 2).contiguous()
406
+ o = _E2EGatherAttnFn.apply(q, k, v, sel, blk_bias_dense, block_size, scale, local_window)
407
+ return o.transpose(1, 2) # [B, HQ, T, D]
sparse_gather_triton.py ADDED
@@ -0,0 +1,524 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """fused Triton kernels for gather-style block-sparse attention
2
+ (per-token top-k kv blocks, single shared index head).
3
+
4
+ Adapted from flash-linear-attention's NSA kernels (fla/ops/nsa/parallel.py), with
5
+ three modifications for this model:
6
+ 1. Small GQA group support: HQ/Hkv = 2 here, but tl.dot needs M >= 16, so the
7
+ head-group tile is padded to GP=16 with masked loads/stores (the kernel is
8
+ memory-bound on K/V block loads, the padded tensor-core lanes are ~free).
9
+ 2. Multi index-head support ([B, HI, T, S] int32, -1 = pad). Each main-attn head
10
+ hm uses index head hm // (HQ // HI); equivalently kv head i_h uses index head
11
+ (i_h * G) // (HQ // HI). HI == 1 reduces to the shared-index-head case.
12
+ Requires num_kv_heads % HI == 0 so a kv-head tile maps to one index head.
13
+ 3. No varlen / block_counts support (single contiguous causal sequence; the
14
+ Python wrapper falls back to the eager gather path otherwise).
15
+
16
+ Semantics match Qwen3Indexer.gather_sparse_attention's eager path exactly:
17
+ per-token topk blocks, token-level causality inside blocks, duplicated block ids
18
+ collapsed (deduped to -1 before the kernel), rows with an empty candidate set
19
+ output zeros.
20
+ """
21
+
22
+ import torch
23
+ import triton
24
+ import triton.language as tl
25
+
26
+ # exported sentinel so modeling_unlimitedocr.py can do a top-level relative
27
+ # import (``from .sparse_gather_triton import SPARSE_BLOCK_SIZE_SENTINEL``) and
28
+ # thereby force transformers' dynamic-module loader to copy this file to the HF
29
+ # modules cache. The lazy imports inside modeling_deepseekv2 are guarded by env
30
+ # flags + try/except and thus not detected as *direct* relative imports of the
31
+ # entry module.
32
+ SPARSE_BLOCK_SIZE_SENTINEL = 128
33
+
34
+
35
+ @triton.jit(do_not_specialize=['T'])
36
+ def _gather_attn_fwd_kernel(
37
+ q, k, v, o, lse, scale, block_indices, seg,
38
+ T,
39
+ H: tl.constexpr, HQ: tl.constexpr, G: tl.constexpr, GP: tl.constexpr,
40
+ K: tl.constexpr, V: tl.constexpr, S: tl.constexpr,
41
+ BS: tl.constexpr, BK: tl.constexpr, BV: tl.constexpr,
42
+ LW: tl.constexpr, NLB: tl.constexpr,
43
+ HI: tl.constexpr, IDXG: tl.constexpr,
44
+ ):
45
+ i_t, i_v, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
46
+ i_b, i_h = i_bh // H, i_bh % H
47
+ bos = i_b * T
48
+ # seg_start = global start index of this query's sample segment
49
+ # (0 for non-packed rows -> identical to the original single-sequence path).
50
+ # Restricting attended keys to [seg_start, i_t] gives per-sample isolation.
51
+ seg_start = tl.load(seg + bos + i_t).to(tl.int32)
52
+
53
+ k += (bos * H + i_h) * K
54
+ v += (bos * H + i_h) * V
55
+ # block_indices: [B, HI, T, S]; select this kv-head's index head.
56
+ i_idx = (i_h * G) // IDXG
57
+ block_indices += ((i_b * HI + i_idx) * T + i_t) * S
58
+
59
+ o_g = tl.arange(0, GP)
60
+ o_k = tl.arange(0, BK)
61
+ o_v = tl.arange(0, BV)
62
+ m_g = o_g < G
63
+ m_k = o_k < K
64
+ m_v = (i_v * BV + o_v) < V
65
+
66
+ p_q = q + (bos + i_t) * HQ * K + (i_h * G + o_g)[:, None] * K + o_k[None, :]
67
+ b_q = tl.load(p_q, mask=m_g[:, None] & m_k[None, :], other=0.)
68
+ b_q = (b_q * scale).to(b_q.dtype)
69
+
70
+ b_o = tl.zeros([GP, BV], dtype=tl.float32)
71
+ b_m = tl.full([GP], float('-inf'), dtype=tl.float32)
72
+ b_acc = tl.zeros([GP], dtype=tl.float32)
73
+ # local window is [local_start, i_t] with local_start = i_t - LW + 1,
74
+ # LW = sparse_local_block * BS (was hardcoded to BS).
75
+ # Selected (non-local) blocks contribute only keys < local_start; the local window is
76
+ # attended separately below. The two key sets are disjoint -> single online softmax over
77
+ # their union == build_block_mask's `block_keep | local_keep` (token-level, exact).
78
+ ls = tl.maximum(i_t - LW + 1, seg_start)
79
+ # --- remote: selected non-local blocks (keys strictly below the local window) ---
80
+ for i in range(S):
81
+ i_s = tl.load(block_indices + i).to(tl.int32) * BS
82
+ if i_s <= i_t and i_s >= 0:
83
+ p_k = tl.make_block_ptr(k, (K, T), (1, H * K), (0, i_s), (BK, BS), (0, 1))
84
+ p_v = tl.make_block_ptr(v, (T, V), (H * V, 1), (i_s, i_v * BV), (BS, BV), (1, 0))
85
+ b_k = tl.load(p_k, boundary_check=(0, 1))
86
+ b_v = tl.load(p_v, boundary_check=(0, 1))
87
+ b_s = tl.dot(b_q, b_k)
88
+ key = i_s + tl.arange(0, BS)
89
+ b_s = tl.where(((i_t >= key) & (key < ls) & (key >= seg_start))[None, :], b_s, float('-inf'))
90
+ b_m, b_mp = tl.maximum(b_m, tl.max(b_s, 1)), b_m
91
+ b_r = tl.exp(b_mp - b_m)
92
+ b_p = tl.exp(b_s - b_m[:, None])
93
+ b_acc = b_acc * b_r + tl.sum(b_p, 1)
94
+ b_o = b_o * b_r[:, None] + tl.dot(b_p.to(b_q.dtype), b_v)
95
+ # --- local: the (<=NLB) physical blocks covering [local_start, i_t] (keys >= local_start) ---
96
+ q_blk = i_t // BS
97
+ for lb in range(NLB):
98
+ i_s = (q_blk - lb) * BS
99
+ if i_s >= 0:
100
+ p_k = tl.make_block_ptr(k, (K, T), (1, H * K), (0, i_s), (BK, BS), (0, 1))
101
+ p_v = tl.make_block_ptr(v, (T, V), (H * V, 1), (i_s, i_v * BV), (BS, BV), (1, 0))
102
+ b_k = tl.load(p_k, boundary_check=(0, 1))
103
+ b_v = tl.load(p_v, boundary_check=(0, 1))
104
+ b_s = tl.dot(b_q, b_k)
105
+ key = i_s + tl.arange(0, BS)
106
+ b_s = tl.where(((i_t >= key) & (key >= ls))[None, :], b_s, float('-inf'))
107
+ b_m, b_mp = tl.maximum(b_m, tl.max(b_s, 1)), b_m
108
+ b_r = tl.exp(b_mp - b_m)
109
+ b_p = tl.exp(b_s - b_m[:, None])
110
+ b_acc = b_acc * b_r + tl.sum(b_p, 1)
111
+ b_o = b_o * b_r[:, None] + tl.dot(b_p.to(b_q.dtype), b_v)
112
+
113
+ # rows with an empty candidate set: b_acc == 0 -> output zeros (not NaN)
114
+ b_safe = tl.where(b_acc > 0, b_acc, 1.0)
115
+ b_o = b_o / b_safe[:, None]
116
+ b_lse = b_m + tl.log(b_safe)
117
+
118
+ p_o = o + (bos + i_t) * HQ * V + (i_h * G + o_g)[:, None] * V + (i_v * BV + o_v)[None, :]
119
+ tl.store(p_o, b_o.to(q.dtype.element_ty), mask=m_g[:, None] & m_v[None, :])
120
+ if i_v == 0:
121
+ p_lse = lse + (bos + i_t) * HQ + i_h * G + o_g
122
+ tl.store(p_lse, b_lse, mask=m_g)
123
+
124
+
125
+ @triton.jit(do_not_specialize=['T'])
126
+ def _gather_attn_bwd_dq_kernel(
127
+ q, k, v, lse, delta, do, dq, scale, block_indices, seg,
128
+ T,
129
+ H: tl.constexpr, HQ: tl.constexpr, G: tl.constexpr, GP: tl.constexpr,
130
+ K: tl.constexpr, V: tl.constexpr, S: tl.constexpr,
131
+ BS: tl.constexpr, BK: tl.constexpr, BV: tl.constexpr,
132
+ LW: tl.constexpr, NLB: tl.constexpr,
133
+ HI: tl.constexpr, IDXG: tl.constexpr,
134
+ ):
135
+ i_t, i_v, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
136
+ i_b, i_h = i_bh // H, i_bh % H
137
+ bos = i_b * T
138
+ seg_start = tl.load(seg + bos + i_t).to(tl.int32)
139
+
140
+ k += (bos * H + i_h) * K
141
+ v += (bos * H + i_h) * V
142
+ i_idx = (i_h * G) // IDXG
143
+ block_indices += ((i_b * HI + i_idx) * T + i_t) * S
144
+
145
+ o_g = tl.arange(0, GP)
146
+ o_k = tl.arange(0, BK)
147
+ o_v = tl.arange(0, BV)
148
+ m_g = o_g < G
149
+ m_k = o_k < K
150
+ m_v = (i_v * BV + o_v) < V
151
+
152
+ p_q = q + (bos + i_t) * HQ * K + (i_h * G + o_g)[:, None] * K + o_k[None, :]
153
+ b_q = tl.load(p_q, mask=m_g[:, None] & m_k[None, :], other=0.)
154
+ b_q = (b_q * scale).to(b_q.dtype)
155
+
156
+ p_do = do + (bos + i_t) * HQ * V + (i_h * G + o_g)[:, None] * V + (i_v * BV + o_v)[None, :]
157
+ b_do = tl.load(p_do, mask=m_g[:, None] & m_v[None, :], other=0.)
158
+ p_lse = lse + (bos + i_t) * HQ + i_h * G + o_g
159
+ # padded head rows: lse=+inf -> p = exp(s - inf) = 0, contributes nothing
160
+ b_lse = tl.load(p_lse, mask=m_g, other=float('inf'))
161
+ p_delta = delta + (bos + i_t) * HQ + i_h * G + o_g
162
+ b_delta = tl.load(p_delta, mask=m_g, other=0.)
163
+
164
+ b_dq = tl.zeros([GP, BK], dtype=tl.float32)
165
+ ls = tl.maximum(i_t - LW + 1, seg_start)
166
+ # --- remote: selected non-local blocks (keys strictly below the local window) ---
167
+ for i in range(S):
168
+ i_s = tl.load(block_indices + i).to(tl.int32) * BS
169
+ if i_s <= i_t and i_s >= 0:
170
+ p_k = tl.make_block_ptr(k, (K, T), (1, H * K), (0, i_s), (BK, BS), (0, 1))
171
+ p_v = tl.make_block_ptr(v, (V, T), (1, H * V), (i_v * BV, i_s), (BV, BS), (0, 1))
172
+ b_k = tl.load(p_k, boundary_check=(0, 1))
173
+ b_v = tl.load(p_v, boundary_check=(0, 1))
174
+ b_s = tl.dot(b_q, b_k)
175
+ key = i_s + tl.arange(0, BS)
176
+ b_p = tl.exp(b_s - b_lse[:, None])
177
+ b_p = tl.where(((i_t >= key) & (key < ls) & (key >= seg_start))[None, :], b_p, 0.)
178
+ b_dp = tl.dot(b_do, b_v)
179
+ b_ds = b_p * (b_dp.to(tl.float32) - b_delta[:, None])
180
+ b_dq += tl.dot(b_ds.to(b_k.dtype), tl.trans(b_k))
181
+ # --- local: the (<=NLB) physical blocks covering [local_start, i_t] (keys >= local_start) ---
182
+ q_blk = i_t // BS
183
+ for lb in range(NLB):
184
+ i_s = (q_blk - lb) * BS
185
+ if i_s >= 0:
186
+ p_k = tl.make_block_ptr(k, (K, T), (1, H * K), (0, i_s), (BK, BS), (0, 1))
187
+ p_v = tl.make_block_ptr(v, (V, T), (1, H * V), (i_v * BV, i_s), (BV, BS), (0, 1))
188
+ b_k = tl.load(p_k, boundary_check=(0, 1))
189
+ b_v = tl.load(p_v, boundary_check=(0, 1))
190
+ b_s = tl.dot(b_q, b_k)
191
+ key = i_s + tl.arange(0, BS)
192
+ b_p = tl.exp(b_s - b_lse[:, None])
193
+ b_p = tl.where(((i_t >= key) & (key >= ls))[None, :], b_p, 0.)
194
+ b_dp = tl.dot(b_do, b_v)
195
+ b_ds = b_p * (b_dp.to(tl.float32) - b_delta[:, None])
196
+ b_dq += tl.dot(b_ds.to(b_k.dtype), tl.trans(b_k))
197
+ b_dq *= scale
198
+
199
+ p_dq = dq + (bos + i_t) * HQ * K + (i_h * G + o_g)[:, None] * K + o_k[None, :]
200
+ tl.store(p_dq, b_dq.to(q.dtype.element_ty), mask=m_g[:, None] & m_k[None, :])
201
+
202
+
203
+ @triton.jit(do_not_specialize=['T'])
204
+ def _gather_attn_bwd_dkv_kernel(
205
+ q, k, v, lse, delta, do, dk, dv, block_mask, seg, scale,
206
+ T,
207
+ H: tl.constexpr, HQ: tl.constexpr, G: tl.constexpr, GP: tl.constexpr,
208
+ K: tl.constexpr, V: tl.constexpr, M: tl.constexpr,
209
+ BS: tl.constexpr, BK: tl.constexpr, BV: tl.constexpr,
210
+ LW: tl.constexpr,
211
+ HI: tl.constexpr, IDXG: tl.constexpr,
212
+ ):
213
+ i_v, i_s, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
214
+ i_b, i_h = i_bh // H, i_bh % H
215
+ bos = i_b * T
216
+ i_idx = (i_h * G) // IDXG
217
+
218
+ p_k = tl.make_block_ptr(k + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_s * BS, 0), (BS, BK), (1, 0))
219
+ p_v = tl.make_block_ptr(v + (bos * H + i_h) * V, (T, V), (H * V, 1), (i_s * BS, i_v * BV), (BS, BV), (1, 0))
220
+ p_dk = tl.make_block_ptr(dk + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_s * BS, 0), (BS, BK), (1, 0))
221
+ p_dv = tl.make_block_ptr(dv + (bos * H + i_h) * V, (T, V), (H * V, 1), (i_s * BS, i_v * BV), (BS, BV), (1, 0))
222
+
223
+ o_g = tl.arange(0, GP)
224
+ o_k = tl.arange(0, BK)
225
+ o_v = tl.arange(0, BV)
226
+ m_g = o_g < G
227
+ m_k = o_k < K
228
+ m_v = (i_v * BV + o_v) < V
229
+
230
+ # [BS, BK]
231
+ b_k = tl.load(p_k, boundary_check=(0, 1))
232
+ b_dk = tl.zeros([BS, BK], dtype=tl.float32)
233
+ # [BS, BV]
234
+ b_v = tl.load(p_v, boundary_check=(0, 1))
235
+ b_dv = tl.zeros([BS, BV], dtype=tl.float32)
236
+
237
+ for i in range(i_s * BS, T):
238
+ # block_mask: [B, HI, T, M] (selected NON-LOCAL blocks only), per index head.
239
+ b_flag = tl.load(block_mask + ((i_b * HI + i_idx) * T + i) * M + i_s)
240
+ # process query i for this kv block if the block is selected (b_flag)
241
+ # OR the block overlaps query i's local window [i - LW + 1, i]. The block spans
242
+ # keys [i_s*BS, (i_s+1)*BS), so overlap implies i < (i_s+1)*BS + LW (loose bound;
243
+ # the exact per-key filter is `keep` below).
244
+ _proc = (b_flag != 0) | (i < (i_s + 1) * BS + LW)
245
+ if _proc:
246
+ p_q = q + (bos + i) * HQ * K + (i_h * G + o_g)[:, None] * K + o_k[None, :]
247
+ b_q = tl.load(p_q, mask=m_g[:, None] & m_k[None, :], other=0.)
248
+ b_q = (b_q * scale).to(b_q.dtype)
249
+
250
+ p_do = do + (bos + i) * HQ * V + (i_h * G + o_g)[:, None] * V + (i_v * BV + o_v)[None, :]
251
+ b_do = tl.load(p_do, mask=m_g[:, None] & m_v[None, :], other=0.)
252
+ p_lse = lse + (bos + i) * HQ + i_h * G + o_g
253
+ b_lse = tl.load(p_lse, mask=m_g, other=float('inf'))
254
+ p_delta = delta + (bos + i) * HQ + i_h * G + o_g
255
+ b_delta = tl.load(p_delta, mask=m_g, other=0.)
256
+
257
+ # [BS, GP]
258
+ b_s = tl.dot(b_k, tl.trans(b_q))
259
+ b_p = tl.exp(b_s - b_lse[None, :])
260
+ # keep = causal & (local-window | selected-block); matches the fwd union.
261
+ # also require the key to be in query i's segment (key >=
262
+ # seg_start_i) so a boundary-straddling / cross-sample block is trimmed.
263
+ seg_i = tl.load(seg + bos + i).to(tl.int32)
264
+ key = i_s * BS + tl.arange(0, BS) # [BS]
265
+ causal = i >= key # [BS]
266
+ in_seg = key >= seg_i # [BS]
267
+ is_local = key > (i - LW) # [BS]
268
+ keep = causal & in_seg & (is_local | (b_flag != 0))
269
+ b_p = tl.where(keep[:, None], b_p, 0.)
270
+ # [BS, GP] @ [GP, BV] -> [BS, BV]
271
+ b_dv += tl.dot(b_p.to(b_do.dtype), b_do)
272
+ # [BS, BV] @ [BV, GP] -> [BS, GP]
273
+ b_dp = tl.dot(b_v, tl.trans(b_do))
274
+ b_ds = b_p * (b_dp.to(tl.float32) - b_delta[None, :])
275
+ # [BS, GP] @ [GP, BK] -> [BS, BK]
276
+ b_dk += tl.dot(b_ds.to(b_q.dtype), b_q)
277
+
278
+ tl.store(p_dk, b_dk.to(q.dtype.element_ty), boundary_check=(0, 1))
279
+ tl.store(p_dv, b_dv.to(q.dtype.element_ty), boundary_check=(0, 1))
280
+
281
+
282
+ def _kernel_meta(q, k, v, sel, block_size):
283
+ B, T, HQ, K = q.shape
284
+ H = k.shape[2]
285
+ V = v.shape[3]
286
+ S = sel.shape[-1]
287
+ G = HQ // H
288
+ GP = max(16, triton.next_power_of_2(G))
289
+ BK = triton.next_power_of_2(K)
290
+ BV = min(128, triton.next_power_of_2(V))
291
+ if BV < V:
292
+ raise NotImplementedError(f"head_dim V={V} > 128 not supported")
293
+ if BK > 256:
294
+ raise NotImplementedError(f"head_dim K={K} > 256 not supported")
295
+ return B, T, HQ, H, K, V, S, G, GP, block_size, BK, BV
296
+
297
+
298
+ def _local_meta(block_size, local_window):
299
+ """local window length in tokens + how many physical blocks can overlap it.
300
+ The window [pos - LW + 1, pos] is not block-aligned, so it spans at most
301
+ ceil(LW / BS) + 1 blocks; with LW a multiple of BS that is LW // BS + 1."""
302
+ lw = block_size if local_window is None else int(local_window)
303
+ lw = max(1, lw)
304
+ nlb = (lw + block_size - 1) // block_size + 1
305
+ return lw, nlb
306
+
307
+
308
+ def _gather_attn_fwd(q, k, v, sel, seg, block_size, scale, local_window=None):
309
+ B, T, HQ, H, K, V, S, G, GP, BS, BK, BV = _kernel_meta(q, k, v, sel, block_size)
310
+ HI = sel.shape[1] # number of index heads; sel is [B, HI, T, S]
311
+ IDXG = HQ // HI # main-attn heads per index head
312
+ LW, NLB = _local_meta(BS, local_window)
313
+ o = torch.empty(B, T, HQ, V, dtype=q.dtype, device=q.device)
314
+ lse = torch.empty(B, T, HQ, dtype=torch.float32, device=q.device)
315
+ _gather_attn_fwd_kernel[(T, 1, B * H)](
316
+ q, k, v, o, lse, scale, sel, seg,
317
+ T=T, H=H, HQ=HQ, G=G, GP=GP, K=K, V=V, S=S, BS=BS, BK=BK, BV=BV,
318
+ LW=LW, NLB=NLB, HI=HI, IDXG=IDXG,
319
+ num_warps=4,
320
+ )
321
+ return o, lse
322
+
323
+
324
+ def _gather_attn_bwd(q, k, v, o, lse, do, sel, seg, block_size, scale, local_window=None):
325
+ B, T, HQ, H, K, V, S, G, GP, BS, BK, BV = _kernel_meta(q, k, v, sel, block_size)
326
+ HI = sel.shape[1]
327
+ IDXG = HQ // HI
328
+ LW, NLB = _local_meta(BS, local_window)
329
+ delta = (o.float() * do.float()).sum(-1) # [B, T, HQ]
330
+
331
+ dq = torch.empty_like(q)
332
+ _gather_attn_bwd_dq_kernel[(T, 1, B * H)](
333
+ q, k, v, lse, delta, do, dq, scale, sel, seg,
334
+ T=T, H=H, HQ=HQ, G=G, GP=GP, K=K, V=V, S=S, BS=BS, BK=BK, BV=BV,
335
+ LW=LW, NLB=NLB, HI=HI, IDXG=IDXG,
336
+ num_warps=4,
337
+ )
338
+
339
+ M = triton.cdiv(T, BS)
340
+ # per index head: [B, HI, T, M]; -1 slots routed to an overflow column then dropped
341
+ block_mask = torch.zeros(B, HI, T, M + 1, dtype=torch.bool, device=q.device)
342
+ block_mask.scatter_(-1, sel.long().clamp(min=0).masked_fill(sel < 0, M), True)
343
+ block_mask = block_mask[..., :M].contiguous()
344
+
345
+ dk = torch.empty_like(k)
346
+ dv = torch.empty_like(v)
347
+ _gather_attn_bwd_dkv_kernel[(1, M, B * H)](
348
+ q, k, v, lse, delta, do, dk, dv, block_mask, seg, scale,
349
+ T=T, H=H, HQ=HQ, G=G, GP=GP, K=K, V=V, M=M, BS=BS, BK=BK, BV=BV,
350
+ LW=LW, HI=HI, IDXG=IDXG,
351
+ num_warps=4,
352
+ )
353
+ return dq, dk, dv
354
+
355
+
356
+ class _TritonGatherAttnFn(torch.autograd.Function):
357
+ """Note: the decoder layer sits under gradient checkpointing, so the tensors
358
+ saved here (incl. o/lse) only live transiently during that layer's recompute
359
+ +backward window -- not across the whole model."""
360
+
361
+ @staticmethod
362
+ def forward(ctx, q, k, v, sel, seg, block_size, scale, local_window=None):
363
+ o, lse = _gather_attn_fwd(q, k, v, sel, seg, block_size, scale, local_window)
364
+ ctx.save_for_backward(q, k, v, sel, seg, o, lse)
365
+ ctx.block_size = block_size
366
+ ctx.scale = scale
367
+ ctx.local_window = local_window
368
+ return o
369
+
370
+ @staticmethod
371
+ def backward(ctx, do):
372
+ q, k, v, sel, seg, o, lse = ctx.saved_tensors
373
+ dq, dk, dv = _gather_attn_bwd(
374
+ q, k, v, o, lse, do.contiguous(), sel, seg, ctx.block_size, ctx.scale,
375
+ ctx.local_window)
376
+ return dq, dk, dv, None, None, None, None, None
377
+
378
+
379
+ def triton_gather_attention(
380
+ query_states, # [B, HQ, T, D]
381
+ key_states, # [B, Hkv, Sk, D]
382
+ value_states, # [B, Hkv, Sk, D]
383
+ block_indices, # [B, H_idx, T, S], -1 = padding
384
+ position_ids, # [B, T]
385
+ attention_mask,
386
+ block_size,
387
+ scale,
388
+ local_window=None,
389
+ ):
390
+ """Fused path entry. Returns [B, HQ, T, D], or None when this input shape is
391
+ unsupported (caller must fall back to the eager gather path):
392
+ padding/attention mask present, prefix kv cache (q_len != k_len), or
393
+ index-head count not dividing the head layout. Supports HI >= 1 index heads
394
+ ([B, HI, T, S]).
395
+
396
+ supports sequence packing. A per-token seg_start (global start
397
+ index of each token's sample segment) is derived as ``arange(T) - position_ids``
398
+ (0 for a non-packed plain-arange row) and threaded into the kernels so every
399
+ query only attends within [seg_start, t] -> no cross-sample leakage."""
400
+ if attention_mask is not None:
401
+ return None
402
+ B, HQ, T, D = query_states.shape
403
+ HI = block_indices.shape[1]
404
+ Hkv = key_states.shape[1]
405
+ # each kv-head tile must map to a single index head (HI | Hkv), and index
406
+ # heads must tile the query heads evenly (HI | HQ).
407
+ if HI < 1 or HQ % HI != 0 or Hkv % HI != 0:
408
+ return None
409
+ if key_states.shape[2] != T:
410
+ return None
411
+ if position_ids is None or position_ids.dim() != 2 or position_ids.shape[-1] != T:
412
+ return None
413
+ # seg_start[b, t] = t - position_ids[b, t]. For a plain-arange
414
+ # (non-packed) row this is all zeros -> kernels reduce to the original path.
415
+ ar = torch.arange(T, device=position_ids.device)
416
+ seg = (ar[None, :] - position_ids).to(torch.int32).contiguous()
417
+
418
+ sel = block_indices # [B, HI, T, S]
419
+ # dedup repeated block ids within a row (duplicates would double-count
420
+ # their softmax mass in the online-softmax accumulation)
421
+ valid = sel >= 0
422
+ sel_sorted, order = sel.sort(dim=-1)
423
+ dup_sorted = torch.zeros_like(valid)
424
+ dup_sorted[..., 1:] = sel_sorted[..., 1:] == sel_sorted[..., :-1]
425
+ dup = torch.zeros_like(valid)
426
+ dup.scatter_(-1, order, dup_sorted)
427
+ sel = sel.masked_fill(dup & valid, -1)
428
+ sel = sel.to(torch.int32).contiguous()
429
+
430
+ q = query_states.transpose(1, 2).contiguous() # [B, T, HQ, D]
431
+ k = key_states.transpose(1, 2).contiguous() # [B, T, Hkv, D]
432
+ v = value_states.transpose(1, 2).contiguous()
433
+ o = _TritonGatherAttnFn.apply(q, k, v, sel, seg, block_size, scale, local_window)
434
+ return o.transpose(1, 2) # [B, HQ, T, D]
435
+
436
+
437
+ # ---------------------------------------------------------------------------
438
+ # fused "QK score + block-amax" for the indexer block selection.
439
+ # Produces block_scores[b, c, blk] = max over keys j in block of (q[b,c] . k[b,j])
440
+ # restricted to NON-LOCAL causal keys, i.e. j < local_start[c] = max(0, pos[c]-local_window+1)
441
+ # (this matches the eager path's `token_future | local_token` masking before the block amax).
442
+ # No dense [B, C, Sk] score tensor is materialised. Non-differentiable (selection is
443
+ # argmax/topk); unscaled to match the eager path. Single index head only.
444
+ # ---------------------------------------------------------------------------
445
+ @triton.jit
446
+ def _block_max_score_kernel(
447
+ q_ptr, k_ptr, pos_ptr, seg_ptr, out_ptr,
448
+ C, Sk,
449
+ sqb, sqc, sqd, skb, sks, skd, spb, spc, sgb, sgc, sob, soc, son,
450
+ D: tl.constexpr, BS: tl.constexpr, BC: tl.constexpr, BD: tl.constexpr,
451
+ LW: tl.constexpr,
452
+ ):
453
+ i_c = tl.program_id(0) # query tile
454
+ i_blk = tl.program_id(1) # key block
455
+ i_b = tl.program_id(2) # batch
456
+
457
+ off_c = i_c * BC + tl.arange(0, BC) # [BC] query rows
458
+ off_d = tl.arange(0, BD) # [BD] head dim
459
+ m_c = off_c < C
460
+ m_d = off_d < D
461
+
462
+ # [BC, BD] query tile
463
+ b_q = tl.load(q_ptr + i_b * sqb + off_c[:, None] * sqc + off_d[None, :] * sqd,
464
+ mask=m_c[:, None] & m_d[None, :], other=0.0)
465
+ # [BC] query positions (GLOBAL under packing) -> non-local upper bound
466
+ # local_start = max(0, pos-local_window+1); seg_start = per-query segment start
467
+ # (0 non-packed) -> remote keys restricted to [seg_start, local_start).
468
+ b_pos = tl.load(pos_ptr + i_b * spb + off_c * spc, mask=m_c, other=0).to(tl.int32)
469
+ b_seg = tl.load(seg_ptr + i_b * sgb + off_c * sgc, mask=m_c, other=0).to(tl.int32)
470
+ b_ls = tl.maximum(b_pos - (LW - 1), 0)
471
+
472
+ j = i_blk * BS + tl.arange(0, BS) # [BS] key indices for this block
473
+ m_j = j < Sk
474
+ # [BS, BD] key block
475
+ b_k = tl.load(k_ptr + i_b * skb + j[:, None] * sks + off_d[None, :] * skd,
476
+ mask=m_j[:, None] & m_d[None, :], other=0.0)
477
+
478
+ # [BC, BS] raw scores (fp32 accumulate); keep only non-local causal in-range,
479
+ # same-segment keys
480
+ b_s = tl.dot(b_q, tl.trans(b_k))
481
+ valid = (j[None, :] < b_ls[:, None]) & (j[None, :] >= b_seg[:, None]) & m_j[None, :]
482
+ b_s = tl.where(valid, b_s, float('-inf'))
483
+ b_max = tl.max(b_s, axis=1) # [BC] block max per query
484
+
485
+ tl.store(out_ptr + i_b * sob + off_c * soc + i_blk * son, b_max, mask=m_c)
486
+
487
+
488
+ def triton_block_max_score(q, k, pos, seg, block_size, num_key_blocks, local_window=None):
489
+ """Fused block-level max of q.k over NON-LOCAL causal SAME-SEGMENT keys; no dense
490
+ [B,C,Sk] tensor.
491
+
492
+ q: [B, C, D] (index-branch queries, single index head)
493
+ k: [B, Sk, D] (index-branch keys)
494
+ pos: [B, C] GLOBAL query positions
495
+ seg: [B, C] per-query segment start (0 for non-packed)
496
+ Returns block_scores [B, C, num_key_blocks] float32 (-inf where a block has no
497
+ valid key). Matches the eager `token_future | local_token | ~same_seg` masking."""
498
+ assert q.dim() == 3 and k.dim() == 3, "expects [B, C, D] / [B, Sk, D]"
499
+ B, C, D = q.shape
500
+ Sk = k.shape[1]
501
+ nblk = num_key_blocks
502
+ BD = triton.next_power_of_2(D)
503
+ if BD > 256:
504
+ raise NotImplementedError(f"index dim D={D} > 256 not supported")
505
+ BC = 64
506
+ q = q.contiguous()
507
+ k = k.contiguous()
508
+ pos = pos.contiguous().to(torch.int32)
509
+ seg = seg.contiguous().to(torch.int32)
510
+ out = torch.empty(B, C, nblk, dtype=torch.float32, device=q.device)
511
+ grid = (triton.cdiv(C, BC), nblk, B)
512
+ _block_max_score_kernel[grid](
513
+ q, k, pos, seg, out,
514
+ C, Sk,
515
+ q.stride(0), q.stride(1), q.stride(2),
516
+ k.stride(0), k.stride(1), k.stride(2),
517
+ pos.stride(0), pos.stride(1),
518
+ seg.stride(0), seg.stride(1),
519
+ out.stride(0), out.stride(1), out.stride(2),
520
+ D=D, BS=block_size, BC=BC, BD=BD,
521
+ LW=(block_size if local_window is None else max(1, int(local_window))),
522
+ num_warps=4,
523
+ )
524
+ return out
special_tokens_map.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|User|>",
4
+ "<|Assistant|>"
5
+ ],
6
+ "bos_token": {
7
+ "content": "<|begin▁of▁sentence|>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false
12
+ },
13
+ "eos_token": {
14
+ "content": "<|end▁of▁sentence|>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false
19
+ },
20
+ "pad_token": {
21
+ "content": "<|▁pad▁|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false
26
+ }
27
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff
 
unlimited_ocr_config_runtime.egg-info/PKG-INFO ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ Metadata-Version: 2.4
2
+ Name: unlimited-ocr-config-runtime
3
+ Version: 0.1.0
4
+ Requires-Python: >=3.10
unlimited_ocr_config_runtime.egg-info/SOURCES.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ pyproject.toml
2
+ unlimited_ocr_config_runtime.egg-info/PKG-INFO
3
+ unlimited_ocr_config_runtime.egg-info/SOURCES.txt
4
+ unlimited_ocr_config_runtime.egg-info/dependency_links.txt
5
+ unlimited_ocr_config_runtime.egg-info/top_level.txt
6
+ unlimited_ocr_runtime/__init__.py
7
+ unlimited_ocr_runtime/model.py
unlimited_ocr_config_runtime.egg-info/dependency_links.txt ADDED
@@ -0,0 +1 @@
 
 
1
+
unlimited_ocr_config_runtime.egg-info/top_level.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ unlimited_ocr_runtime
unlimited_ocr_runtime/__init__.py ADDED
File without changes
unlimited_ocr_runtime/model.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from copy import deepcopy
2
+ from functools import partial
3
+ from unittest.mock import patch
4
+
5
+ import torch
6
+ from torch import nn
7
+ from sglang.srt.models import deepseek_ocr as vision
8
+ from sglang.srt.models import unlimited_ocr as base
9
+ from sglang.srt.models import unlimited_ocr_v2 as dash
10
+ from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
11
+ from sglang.srt.models.unlimited_ocr_v2 import UnlimitedOCRV2ForCausalLM as UpstreamModel
12
+
13
+
14
+ class ConfigAttention(dash.UnlimitedOCRV2Attention):
15
+ def __init__(self, config, layer_id, quant_config=None, prefix=""):
16
+ # Use the pinned constructor and forward, with a separate residual width.
17
+ attention_config = deepcopy(config)
18
+ attention_config.hidden_size = config.num_attention_heads * config.head_dim
19
+ super().__init__(attention_config, layer_id, quant_config, prefix)
20
+ self.hidden_size = config.hidden_size
21
+ outputs = {
22
+ "q_proj": config.num_attention_heads * self.head_dim,
23
+ "k_proj": config.num_key_value_heads * self.head_dim,
24
+ "v_proj": config.num_key_value_heads * self.head_dim,
25
+ "index_q_proj": self.idx_head_dim,
26
+ "index_k_proj": self.idx_head_dim,
27
+ }
28
+ for name, output_size in outputs.items():
29
+ setattr(self, name, ColumnParallelLinear(config.hidden_size, output_size,
30
+ bias=False, gather_output=name.startswith("index_"), quant_config=quant_config,
31
+ prefix=f"{prefix}.{name}" if prefix else name))
32
+ self.o_proj = RowParallelLinear(config.num_attention_heads * self.head_dim,
33
+ config.hidden_size, bias=False, quant_config=quant_config,
34
+ prefix=f"{prefix}.o_proj" if prefix else "o_proj")
35
+
36
+
37
+ class UnlimitedOCRV2ForCausalLM(UpstreamModel):
38
+ def __init__(self, *, config, quant_config=None, prefix=""):
39
+ specs = config.vision_config.width
40
+
41
+ def sam():
42
+ spec = specs["sam_vit_b"]
43
+ encoder = vision.ImageEncoderViT(embed_dim=spec["width"], depth=spec["layers"],
44
+ num_heads=spec["heads"], global_attn_indexes=spec["global_attn_indexes"],
45
+ out_chans=spec["neck_channels"], window_size=14, use_rel_pos=True,
46
+ norm_layer=partial(nn.LayerNorm, eps=1e-6))
47
+ c1, c2 = spec["downsample_channels"]
48
+ encoder.net_2 = nn.Conv2d(spec["neck_channels"], c1, 3, stride=2, padding=1, bias=False)
49
+ encoder.net_3 = nn.Conv2d(c1, c2, 3, stride=2, padding=1, bias=False)
50
+ return encoder
51
+
52
+ def clip():
53
+ spec = specs["clip-l-14-224"]
54
+ cfg = deepcopy(vision.vit_model_cfg)
55
+ cfg.update(num_layers=spec["layers"], hidden_size=spec["width"], num_heads=spec["heads"],
56
+ num_attention_heads=spec["heads"], ffn_hidden_size=spec["width"] * 4)
57
+ return vision.VitModel(cfg)
58
+
59
+ with (patch.object(base, "build_sam_vit_b", sam),
60
+ patch.object(base, "build_clip_l", clip),
61
+ patch.object(dash, "UnlimitedOCRV2Attention", ConfigAttention)):
62
+ super().__init__(config=config, quant_config=quant_config, prefix=prefix)
63
+
64
+
65
+ EntryClass = UnlimitedOCRV2ForCausalLM