Instructions to use tiny-random/unlimited-ocr-2 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use tiny-random/unlimited-ocr-2 with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="tiny-random/unlimited-ocr-2", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("tiny-random/unlimited-ocr-2", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use tiny-random/unlimited-ocr-2 with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "tiny-random/unlimited-ocr-2" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "tiny-random/unlimited-ocr-2", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/tiny-random/unlimited-ocr-2
- SGLang
How to use tiny-random/unlimited-ocr-2 with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "tiny-random/unlimited-ocr-2" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "tiny-random/unlimited-ocr-2", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "tiny-random/unlimited-ocr-2" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "tiny-random/unlimited-ocr-2", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use tiny-random/unlimited-ocr-2 with Docker Model Runner:
docker model run hf.co/tiny-random/unlimited-ocr-2
Upload folder using huggingface_hub
Browse files- .meta/create_model.py +218 -0
- .meta/env.json +70 -0
- .meta/model.log +140 -0
- README.md +127 -0
- _sparse_base_impl.py +0 -0
- config.json +215 -0
- configuration_deepseek_v2.py +212 -0
- conversation.py +280 -0
- dash_attention.py +1349 -0
- deepencoder.py +1072 -0
- generation_config.json +6 -0
- model.safetensors +3 -0
- modeling_deepseekv2.py +531 -0
- modeling_unlimitedocr.py +1228 -0
- packing_patch.py +168 -0
- pyproject.toml +11 -0
- sparse_gather_e2e_triton.py +407 -0
- sparse_gather_triton.py +524 -0
- special_tokens_map.json +27 -0
- tokenizer.json +0 -0
- tokenizer_config.json +0 -0
- unlimited_ocr_config_runtime.egg-info/PKG-INFO +4 -0
- unlimited_ocr_config_runtime.egg-info/SOURCES.txt +7 -0
- unlimited_ocr_config_runtime.egg-info/dependency_links.txt +1 -0
- unlimited_ocr_config_runtime.egg-info/top_level.txt +1 -0
- unlimited_ocr_runtime/__init__.py +0 -0
- unlimited_ocr_runtime/model.py +65 -0
.meta/create_model.py
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str:
|
| 2 |
+
import json
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
from huggingface_hub import hf_hub_download
|
| 7 |
+
from transformers import AutoConfig, AutoModel, set_seed
|
| 8 |
+
|
| 9 |
+
folder = Path(save_folder)
|
| 10 |
+
folder.mkdir(parents=True, exist_ok=True)
|
| 11 |
+
for name in (
|
| 12 |
+
"configuration_deepseek_v2.py",
|
| 13 |
+
"modeling_unlimitedocr.py",
|
| 14 |
+
"modeling_deepseekv2.py",
|
| 15 |
+
"_sparse_base_impl.py",
|
| 16 |
+
"dash_attention.py",
|
| 17 |
+
"packing_patch.py",
|
| 18 |
+
"sparse_gather_triton.py",
|
| 19 |
+
"sparse_gather_e2e_triton.py",
|
| 20 |
+
"deepencoder.py",
|
| 21 |
+
"conversation.py",
|
| 22 |
+
"tokenizer.json",
|
| 23 |
+
"tokenizer_config.json",
|
| 24 |
+
"special_tokens_map.json",
|
| 25 |
+
"generation_config.json",
|
| 26 |
+
):
|
| 27 |
+
(folder / name).write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
|
| 28 |
+
config = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
|
| 29 |
+
# Keep four TP heads of dimension 32 independently from the residual width.
|
| 30 |
+
# Q/K/V project to 128 attention channels; o_proj returns to hidden_size.
|
| 31 |
+
dimensions = dict(
|
| 32 |
+
hidden_size=32,
|
| 33 |
+
head_dim=32,
|
| 34 |
+
intermediate_size=256,
|
| 35 |
+
moe_intermediate_size=64,
|
| 36 |
+
num_hidden_layers=4,
|
| 37 |
+
num_attention_heads=4,
|
| 38 |
+
num_key_value_heads=4,
|
| 39 |
+
n_routed_experts=16,
|
| 40 |
+
v_head_dim=32,
|
| 41 |
+
)
|
| 42 |
+
config.update(dimensions)
|
| 43 |
+
config["language_config"].update(dimensions)
|
| 44 |
+
sparse = config["sparse_attention_config"]
|
| 45 |
+
sparse["sparse_index_dim"] = 32
|
| 46 |
+
for name in ("sparse_attention_freq", "dash_scale_per_layer", "dash_floor_per_layer"):
|
| 47 |
+
sparse[name] = sparse[name][:4]
|
| 48 |
+
vision = config["vision_config"]["width"]
|
| 49 |
+
# Two complete local/local/global SAM cycles; both encoder families remain.
|
| 50 |
+
vision["sam_vit_b"].update(
|
| 51 |
+
width=32, heads=4, layers=6, global_attn_indexes=[2, 5], neck_channels=8, downsample_channels=[16, 32]
|
| 52 |
+
)
|
| 53 |
+
vision["clip-l-14-224"].update(width=32, heads=4, layers=4)
|
| 54 |
+
config["projector_config"].update(input_dim=64, n_embed=config["hidden_size"])
|
| 55 |
+
|
| 56 |
+
def replace_once(name, old, new):
|
| 57 |
+
path = folder / name
|
| 58 |
+
code = path.read_text()
|
| 59 |
+
if code.count(old) != 1:
|
| 60 |
+
raise ValueError(f"Unexpected upstream implementation in {name}: {old}")
|
| 61 |
+
path.write_text(code.replace(old, new, 1))
|
| 62 |
+
|
| 63 |
+
# Transformers' LlamaAttention already supports an explicit head_dim. DASH
|
| 64 |
+
# must use that same dimension for its attention scaling and Triton kernel.
|
| 65 |
+
replace_once(
|
| 66 |
+
"dash_attention.py",
|
| 67 |
+
"self.head_dim = config.hidden_size // config.num_attention_heads",
|
| 68 |
+
'self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)',
|
| 69 |
+
)
|
| 70 |
+
|
| 71 |
+
# The reference hardcodes encoder widths; expose its existing builders to
|
| 72 |
+
# config without replacing attention, routing, position encodings or APIs.
|
| 73 |
+
replace_once(
|
| 74 |
+
"modeling_unlimitedocr.py",
|
| 75 |
+
"self.sam_model = build_sam_vit_b()",
|
| 76 |
+
"self.sam_model = build_sam_vit_b(config=config)",
|
| 77 |
+
)
|
| 78 |
+
replace_once(
|
| 79 |
+
"modeling_unlimitedocr.py",
|
| 80 |
+
"self.vision_model = build_clip_l()",
|
| 81 |
+
"self.vision_model = build_clip_l(config=config)",
|
| 82 |
+
)
|
| 83 |
+
replace_once("modeling_unlimitedocr.py", " n_embed = 1280", " n_embed = config.hidden_size")
|
| 84 |
+
replace_once(
|
| 85 |
+
"modeling_unlimitedocr.py",
|
| 86 |
+
"input_dim=2048, n_embed=n_embed",
|
| 87 |
+
'input_dim=config.projector_config["input_dim"], n_embed=n_embed',
|
| 88 |
+
)
|
| 89 |
+
replace_once(
|
| 90 |
+
"deepencoder.py",
|
| 91 |
+
"def build_clip_l():",
|
| 92 |
+
"""def build_clip_l(config=None):
|
| 93 |
+
cfg = copy.deepcopy(vit_model_cfg)
|
| 94 |
+
if config is not None:
|
| 95 |
+
spec = config.vision_config['width']['clip-l-14-224']
|
| 96 |
+
cfg.update(num_layers=spec['layers'], hidden_size=spec['width'],
|
| 97 |
+
num_heads=spec['heads'], num_attention_heads=spec['heads'],
|
| 98 |
+
ffn_hidden_size=spec['width'] * 4)""",
|
| 99 |
+
)
|
| 100 |
+
replace_once("deepencoder.py", "cfg=vit_model_cfg,", "cfg=cfg,")
|
| 101 |
+
replace_once(
|
| 102 |
+
"deepencoder.py",
|
| 103 |
+
"def build_sam_vit_b(checkpoint=None):",
|
| 104 |
+
"""def build_sam_vit_b(checkpoint=None, config=None):
|
| 105 |
+
if config is not None:
|
| 106 |
+
spec = config.vision_config['width']['sam_vit_b']
|
| 107 |
+
encoder = ImageEncoderViT(embed_dim=spec['width'], depth=spec['layers'],
|
| 108 |
+
num_heads=spec['heads'], global_attn_indexes=spec['global_attn_indexes'],
|
| 109 |
+
out_chans=spec['neck_channels'], window_size=14, use_rel_pos=True,
|
| 110 |
+
norm_layer=partial(torch.nn.LayerNorm, eps=1e-6))
|
| 111 |
+
c1, c2 = spec['downsample_channels']
|
| 112 |
+
encoder.net_2 = nn.Conv2d(spec['neck_channels'], c1, 3, stride=2, padding=1, bias=False)
|
| 113 |
+
encoder.net_3 = nn.Conv2d(c1, c2, 3, stride=2, padding=1, bias=False)
|
| 114 |
+
return encoder""",
|
| 115 |
+
)
|
| 116 |
+
(folder / "config.json").write_text(json.dumps(config, indent=2) + "\n")
|
| 117 |
+
set_seed(42)
|
| 118 |
+
# This pinned reference registers only mha_eager. Its infer() explicitly
|
| 119 |
+
# runs DASH sparse prefill/decode; the vision encoders use SDPA.
|
| 120 |
+
model = AutoModel.from_config(
|
| 121 |
+
AutoConfig.from_pretrained(folder, trust_remote_code=True), trust_remote_code=True, torch_dtype=torch.bfloat16
|
| 122 |
+
).to(torch.bfloat16)
|
| 123 |
+
model.save_pretrained(folder)
|
| 124 |
+
(folder / "config.json").write_text(json.dumps(config, indent=2) + "\n")
|
| 125 |
+
(folder / "generation_config.json").write_bytes(
|
| 126 |
+
Path(hf_hub_download(source_model_id, "generation_config.json", revision=source_revision)).read_bytes()
|
| 127 |
+
)
|
| 128 |
+
# SGLang's official wheel also hardcodes the full-size vision constructors.
|
| 129 |
+
# Its external-model hook changes only construction; DASH stays upstream.
|
| 130 |
+
runtime = folder / "unlimited_ocr_runtime"
|
| 131 |
+
runtime.mkdir(exist_ok=True)
|
| 132 |
+
(runtime / "__init__.py").write_text("")
|
| 133 |
+
(runtime / "model.py").write_text("""from copy import deepcopy
|
| 134 |
+
from functools import partial
|
| 135 |
+
from unittest.mock import patch
|
| 136 |
+
|
| 137 |
+
import torch
|
| 138 |
+
from torch import nn
|
| 139 |
+
from sglang.srt.models import deepseek_ocr as vision
|
| 140 |
+
from sglang.srt.models import unlimited_ocr as base
|
| 141 |
+
from sglang.srt.models import unlimited_ocr_v2 as dash
|
| 142 |
+
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
|
| 143 |
+
from sglang.srt.models.unlimited_ocr_v2 import UnlimitedOCRV2ForCausalLM as UpstreamModel
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
class ConfigAttention(dash.UnlimitedOCRV2Attention):
|
| 147 |
+
def __init__(self, config, layer_id, quant_config=None, prefix=""):
|
| 148 |
+
# Use the pinned constructor and forward, with a separate residual width.
|
| 149 |
+
attention_config = deepcopy(config)
|
| 150 |
+
attention_config.hidden_size = config.num_attention_heads * config.head_dim
|
| 151 |
+
super().__init__(attention_config, layer_id, quant_config, prefix)
|
| 152 |
+
self.hidden_size = config.hidden_size
|
| 153 |
+
outputs = {
|
| 154 |
+
"q_proj": config.num_attention_heads * self.head_dim,
|
| 155 |
+
"k_proj": config.num_key_value_heads * self.head_dim,
|
| 156 |
+
"v_proj": config.num_key_value_heads * self.head_dim,
|
| 157 |
+
"index_q_proj": self.idx_head_dim,
|
| 158 |
+
"index_k_proj": self.idx_head_dim,
|
| 159 |
+
}
|
| 160 |
+
for name, output_size in outputs.items():
|
| 161 |
+
setattr(self, name, ColumnParallelLinear(config.hidden_size, output_size,
|
| 162 |
+
bias=False, gather_output=name.startswith("index_"), quant_config=quant_config,
|
| 163 |
+
prefix=f"{prefix}.{name}" if prefix else name))
|
| 164 |
+
self.o_proj = RowParallelLinear(config.num_attention_heads * self.head_dim,
|
| 165 |
+
config.hidden_size, bias=False, quant_config=quant_config,
|
| 166 |
+
prefix=f"{prefix}.o_proj" if prefix else "o_proj")
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
class UnlimitedOCRV2ForCausalLM(UpstreamModel):
|
| 170 |
+
def __init__(self, *, config, quant_config=None, prefix=""):
|
| 171 |
+
specs = config.vision_config.width
|
| 172 |
+
|
| 173 |
+
def sam():
|
| 174 |
+
spec = specs["sam_vit_b"]
|
| 175 |
+
encoder = vision.ImageEncoderViT(embed_dim=spec["width"], depth=spec["layers"],
|
| 176 |
+
num_heads=spec["heads"], global_attn_indexes=spec["global_attn_indexes"],
|
| 177 |
+
out_chans=spec["neck_channels"], window_size=14, use_rel_pos=True,
|
| 178 |
+
norm_layer=partial(nn.LayerNorm, eps=1e-6))
|
| 179 |
+
c1, c2 = spec["downsample_channels"]
|
| 180 |
+
encoder.net_2 = nn.Conv2d(spec["neck_channels"], c1, 3, stride=2, padding=1, bias=False)
|
| 181 |
+
encoder.net_3 = nn.Conv2d(c1, c2, 3, stride=2, padding=1, bias=False)
|
| 182 |
+
return encoder
|
| 183 |
+
|
| 184 |
+
def clip():
|
| 185 |
+
spec = specs["clip-l-14-224"]
|
| 186 |
+
cfg = deepcopy(vision.vit_model_cfg)
|
| 187 |
+
cfg.update(num_layers=spec["layers"], hidden_size=spec["width"], num_heads=spec["heads"],
|
| 188 |
+
num_attention_heads=spec["heads"], ffn_hidden_size=spec["width"] * 4)
|
| 189 |
+
return vision.VitModel(cfg)
|
| 190 |
+
|
| 191 |
+
with (patch.object(base, "build_sam_vit_b", sam),
|
| 192 |
+
patch.object(base, "build_clip_l", clip),
|
| 193 |
+
patch.object(dash, "UnlimitedOCRV2Attention", ConfigAttention)):
|
| 194 |
+
super().__init__(config=config, quant_config=quant_config, prefix=prefix)
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
EntryClass = UnlimitedOCRV2ForCausalLM
|
| 198 |
+
""")
|
| 199 |
+
(folder / "pyproject.toml").write_text("""[build-system]
|
| 200 |
+
requires = ["setuptools>=61"]
|
| 201 |
+
build-backend = "setuptools.build_meta"
|
| 202 |
+
|
| 203 |
+
[project]
|
| 204 |
+
name = "unlimited-ocr-config-runtime"
|
| 205 |
+
version = "0.1.0"
|
| 206 |
+
requires-python = ">=3.10"
|
| 207 |
+
|
| 208 |
+
[tool.setuptools.packages.find]
|
| 209 |
+
include = ["unlimited_ocr_runtime*"]
|
| 210 |
+
""")
|
| 211 |
+
return repr(model)
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
create_model(
|
| 215 |
+
source_model_id="baidu/Unlimited-OCR-2",
|
| 216 |
+
source_revision="c14071b549f4fcab10909f3c4acc6a2edb366947",
|
| 217 |
+
save_folder="/tmp/unlimited-ocr-2",
|
| 218 |
+
)
|
.meta/env.json
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"source_model": {
|
| 3 |
+
"name": "baidu/Unlimited-OCR-2",
|
| 4 |
+
"commit_id": "c14071b549f4fcab10909f3c4acc6a2edb366947"
|
| 5 |
+
},
|
| 6 |
+
"environments": {
|
| 7 |
+
"creation": {
|
| 8 |
+
"python": "3.12.13",
|
| 9 |
+
"packages": {
|
| 10 |
+
"addict": "2.4.0",
|
| 11 |
+
"easydict": "1.13",
|
| 12 |
+
"einops": "0.9.0.dev0",
|
| 13 |
+
"huggingface_hub": "0.36.0",
|
| 14 |
+
"kernels": "0.7.0",
|
| 15 |
+
"safetensors": "0.8.0",
|
| 16 |
+
"tokenizers": "0.21.4",
|
| 17 |
+
"torch": "2.13.0+cu126",
|
| 18 |
+
"torchvision": "0.28.0+cu126",
|
| 19 |
+
"transformers": "4.52.1"
|
| 20 |
+
},
|
| 21 |
+
"cuda": "12.6",
|
| 22 |
+
"hardware": {
|
| 23 |
+
"gpu_model": "NVIDIA H20"
|
| 24 |
+
}
|
| 25 |
+
},
|
| 26 |
+
"sglang": {
|
| 27 |
+
"python": "3.12.13",
|
| 28 |
+
"packages": {
|
| 29 |
+
"apache-tvm-ffi": "0.1.11",
|
| 30 |
+
"flashinfer-python": "0.6.18.post1",
|
| 31 |
+
"huggingface_hub": "1.9.0",
|
| 32 |
+
"kernels": "0.14.1",
|
| 33 |
+
"ninja": "1.13.0",
|
| 34 |
+
"safetensors": "0.8.0",
|
| 35 |
+
"sgl-deep-gemm": "0.1.4.post1",
|
| 36 |
+
"sglang": "0.5.20.dev13+g75b9b06b02",
|
| 37 |
+
"sglang-kernel": "0.4.7",
|
| 38 |
+
"tokenizers": "0.22.2",
|
| 39 |
+
"torch": "2.13.0+cu126",
|
| 40 |
+
"transformers": "5.12.1",
|
| 41 |
+
"triton": "3.7.1"
|
| 42 |
+
},
|
| 43 |
+
"cuda": "12.6",
|
| 44 |
+
"hardware": {
|
| 45 |
+
"gpu_model": "NVIDIA H20"
|
| 46 |
+
}
|
| 47 |
+
},
|
| 48 |
+
"transformers": {
|
| 49 |
+
"python": "3.12.13",
|
| 50 |
+
"packages": {
|
| 51 |
+
"Pillow": "12.3.0",
|
| 52 |
+
"addict": "2.4.0",
|
| 53 |
+
"easydict": "1.13",
|
| 54 |
+
"einops": "0.9.0.dev0",
|
| 55 |
+
"huggingface_hub": "0.36.0",
|
| 56 |
+
"kernels": "0.7.0",
|
| 57 |
+
"safetensors": "0.8.0",
|
| 58 |
+
"tokenizers": "0.21.4",
|
| 59 |
+
"torch": "2.13.0+cu126",
|
| 60 |
+
"torchvision": "0.28.0+cu126",
|
| 61 |
+
"transformers": "4.52.1",
|
| 62 |
+
"triton": "3.7.1"
|
| 63 |
+
},
|
| 64 |
+
"cuda": "12.6",
|
| 65 |
+
"hardware": {
|
| 66 |
+
"gpu_model": "NVIDIA H20"
|
| 67 |
+
}
|
| 68 |
+
}
|
| 69 |
+
}
|
| 70 |
+
}
|
.meta/model.log
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
UnlimitedOCRForCausalLM(
|
| 2 |
+
(model): UnlimitedOCRModel(
|
| 3 |
+
(embed_tokens): Embedding(129280, 32, padding_idx=2)
|
| 4 |
+
(layers): ModuleList(
|
| 5 |
+
(0): DeepseekV2DecoderLayer(
|
| 6 |
+
(self_attn): SlidingWindowLlamaAttention(
|
| 7 |
+
(q_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 8 |
+
(k_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 9 |
+
(v_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 10 |
+
(o_proj): Linear(in_features=128, out_features=32, bias=False)
|
| 11 |
+
(rotary_emb): LlamaRotaryEmbedding()
|
| 12 |
+
(indexer): DeepseekV2Indexer(
|
| 13 |
+
(q_proj): Linear(in_features=32, out_features=32, bias=False)
|
| 14 |
+
(k_proj): Linear(in_features=32, out_features=32, bias=False)
|
| 15 |
+
(q_norm): _GemmaRMSNorm()
|
| 16 |
+
(k_norm): _GemmaRMSNorm()
|
| 17 |
+
(rotary_emb_1d): DeepseekV2IndexerRotaryEmbedding()
|
| 18 |
+
(rotary_emb_2d): DeepseekV2IndexerRotary2D()
|
| 19 |
+
(rotary_emb): DeepseekV2IndexerRotary2D()
|
| 20 |
+
)
|
| 21 |
+
(rotary_emb_2d_main): DeepseekV2IndexerRotary2D()
|
| 22 |
+
(dash_router): DashAttention(
|
| 23 |
+
(q_proj): Linear(in_features=32, out_features=32, bias=False)
|
| 24 |
+
(k_proj): Linear(in_features=32, out_features=32, bias=False)
|
| 25 |
+
(q_norm): _RMSNorm()
|
| 26 |
+
(k_norm): _RMSNorm()
|
| 27 |
+
)
|
| 28 |
+
)
|
| 29 |
+
(mlp): DeepseekV2MLP(
|
| 30 |
+
(gate_proj): Linear(in_features=32, out_features=256, bias=False)
|
| 31 |
+
(up_proj): Linear(in_features=32, out_features=256, bias=False)
|
| 32 |
+
(down_proj): Linear(in_features=256, out_features=32, bias=False)
|
| 33 |
+
(act_fn): SiLU()
|
| 34 |
+
)
|
| 35 |
+
(input_layernorm): DeepseekV2RMSNorm()
|
| 36 |
+
(post_attention_layernorm): DeepseekV2RMSNorm()
|
| 37 |
+
)
|
| 38 |
+
(1-3): 3 x DeepseekV2DecoderLayer(
|
| 39 |
+
(self_attn): SlidingWindowLlamaAttention(
|
| 40 |
+
(q_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 41 |
+
(k_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 42 |
+
(v_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 43 |
+
(o_proj): Linear(in_features=128, out_features=32, bias=False)
|
| 44 |
+
(rotary_emb): LlamaRotaryEmbedding()
|
| 45 |
+
(indexer): DeepseekV2Indexer(
|
| 46 |
+
(q_proj): Linear(in_features=32, out_features=32, bias=False)
|
| 47 |
+
(k_proj): Linear(in_features=32, out_features=32, bias=False)
|
| 48 |
+
(q_norm): _GemmaRMSNorm()
|
| 49 |
+
(k_norm): _GemmaRMSNorm()
|
| 50 |
+
(rotary_emb_1d): DeepseekV2IndexerRotaryEmbedding()
|
| 51 |
+
(rotary_emb_2d): DeepseekV2IndexerRotary2D()
|
| 52 |
+
(rotary_emb): DeepseekV2IndexerRotary2D()
|
| 53 |
+
)
|
| 54 |
+
(rotary_emb_2d_main): DeepseekV2IndexerRotary2D()
|
| 55 |
+
(dash_router): DashAttention(
|
| 56 |
+
(q_proj): Linear(in_features=32, out_features=32, bias=False)
|
| 57 |
+
(k_proj): Linear(in_features=32, out_features=32, bias=False)
|
| 58 |
+
(q_norm): _RMSNorm()
|
| 59 |
+
(k_norm): _RMSNorm()
|
| 60 |
+
)
|
| 61 |
+
)
|
| 62 |
+
(mlp): DeepseekV2MoE(
|
| 63 |
+
(experts): ModuleList(
|
| 64 |
+
(0-15): 16 x DeepseekV2MLP(
|
| 65 |
+
(gate_proj): Linear(in_features=32, out_features=64, bias=False)
|
| 66 |
+
(up_proj): Linear(in_features=32, out_features=64, bias=False)
|
| 67 |
+
(down_proj): Linear(in_features=64, out_features=32, bias=False)
|
| 68 |
+
(act_fn): SiLU()
|
| 69 |
+
)
|
| 70 |
+
)
|
| 71 |
+
(gate): MoEGate()
|
| 72 |
+
(shared_experts): DeepseekV2MLP(
|
| 73 |
+
(gate_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 74 |
+
(up_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 75 |
+
(down_proj): Linear(in_features=128, out_features=32, bias=False)
|
| 76 |
+
(act_fn): SiLU()
|
| 77 |
+
)
|
| 78 |
+
)
|
| 79 |
+
(input_layernorm): DeepseekV2RMSNorm()
|
| 80 |
+
(post_attention_layernorm): DeepseekV2RMSNorm()
|
| 81 |
+
)
|
| 82 |
+
)
|
| 83 |
+
(norm): DeepseekV2RMSNorm()
|
| 84 |
+
(sam_model): ImageEncoderViT(
|
| 85 |
+
(patch_embed): PatchEmbed(
|
| 86 |
+
(proj): Conv2d(3, 32, kernel_size=(16, 16), stride=(16, 16))
|
| 87 |
+
)
|
| 88 |
+
(blocks): ModuleList(
|
| 89 |
+
(0-5): 6 x Block(
|
| 90 |
+
(norm1): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
|
| 91 |
+
(attn): Attention(
|
| 92 |
+
(qkv): Linear(in_features=32, out_features=96, bias=True)
|
| 93 |
+
(proj): Linear(in_features=32, out_features=32, bias=True)
|
| 94 |
+
)
|
| 95 |
+
(norm2): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
|
| 96 |
+
(mlp): MLPBlock(
|
| 97 |
+
(lin1): Linear(in_features=32, out_features=128, bias=True)
|
| 98 |
+
(lin2): Linear(in_features=128, out_features=32, bias=True)
|
| 99 |
+
(act): GELU(approximate='none')
|
| 100 |
+
)
|
| 101 |
+
)
|
| 102 |
+
)
|
| 103 |
+
(neck): Sequential(
|
| 104 |
+
(0): Conv2d(32, 8, kernel_size=(1, 1), stride=(1, 1), bias=False)
|
| 105 |
+
(1): LayerNorm2d()
|
| 106 |
+
(2): Conv2d(8, 8, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
|
| 107 |
+
(3): LayerNorm2d()
|
| 108 |
+
)
|
| 109 |
+
(net_2): Conv2d(8, 16, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
|
| 110 |
+
(net_3): Conv2d(16, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
|
| 111 |
+
)
|
| 112 |
+
(vision_model): VitModel(
|
| 113 |
+
(embeddings): CLIPVisionEmbeddings(
|
| 114 |
+
(patch_embedding): Conv2d(3, 32, kernel_size=(14, 14), stride=(14, 14), bias=False)
|
| 115 |
+
(position_embedding): Embedding(257, 32)
|
| 116 |
+
)
|
| 117 |
+
(transformer): NoTPTransformer(
|
| 118 |
+
(layers): ModuleList(
|
| 119 |
+
(0-3): 4 x NoTPTransformerBlock(
|
| 120 |
+
(self_attn): NoTPAttention(
|
| 121 |
+
(qkv_proj): Linear(in_features=32, out_features=96, bias=True)
|
| 122 |
+
(out_proj): Linear(in_features=32, out_features=32, bias=True)
|
| 123 |
+
)
|
| 124 |
+
(mlp): NoTPFeedForward(
|
| 125 |
+
(fc1): Linear(in_features=32, out_features=128, bias=True)
|
| 126 |
+
(fc2): Linear(in_features=128, out_features=32, bias=True)
|
| 127 |
+
)
|
| 128 |
+
(layer_norm1): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
|
| 129 |
+
(layer_norm2): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
|
| 130 |
+
)
|
| 131 |
+
)
|
| 132 |
+
)
|
| 133 |
+
(pre_layrnorm): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
|
| 134 |
+
)
|
| 135 |
+
(projector): MlpProjector(
|
| 136 |
+
(layers): Linear(in_features=64, out_features=32, bias=True)
|
| 137 |
+
)
|
| 138 |
+
)
|
| 139 |
+
(lm_head): Linear(in_features=32, out_features=129280, bias=False)
|
| 140 |
+
)
|
README.md
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
library_name: transformers
|
| 3 |
+
pipeline_tag: image-text-to-text
|
| 4 |
+
base_model:
|
| 5 |
+
- baidu/Unlimited-OCR-2
|
| 6 |
+
---
|
| 7 |
+
|
| 8 |
+
This is a randomly initialized tiny model based on [baidu/Unlimited-OCR-2](https://huggingface.co/baidu/Unlimited-OCR-2) for testing and debugging.
|
| 9 |
+
|
| 10 |
+
| Feature | Source | This repo |
|
| 11 |
+
|---|---|---|
|
| 12 |
+
| Checkpoint tensor elements | 3.34B BF16, 257 I64 | 9.04M BF16, 257 I64 |
|
| 13 |
+
| Safetensors weight files | 6.69 GB (2 files) | 18.12 MB (model.safetensors) |
|
| 14 |
+
| Decoder layers / hidden width / dense MLP | 12 / 1,280 / 6,848 | 4 / 32 / 256 |
|
| 15 |
+
| Q / KV heads / head dimension | 10 / 10 / 128 | 4 / 4 / 32 |
|
| 16 |
+
| Routed experts / top-k / shared experts | 64 / 6 / 2 | 16 / 6 / 2 |
|
| 17 |
+
| Expert intermediate width / initial dense layers | 896 / 1 | 64 / 1 |
|
| 18 |
+
| DASH index width / sparse block size | 128 / 32 | 32 / 32 |
|
| 19 |
+
| Page and token RoPE / sparse layer coverage | page2d / all layers | Same |
|
| 20 |
+
| SAM layers / width / global attention layers | 12 / 768 / 2,5,8,11 | 6 / 32 / 2,5 |
|
| 21 |
+
| CLIP layers / width | 24 / 1,024 | 4 / 32 |
|
| 22 |
+
| SAM neck / downsampling channels | 256 / 512,1,024 | 8 / 16,32 |
|
| 23 |
+
| Vision projector input / output | 2,048 / 1,280 | 64 / 32 |
|
| 24 |
+
| Global / crop image size / page visual tokens | 1,024 / 640 / 273 | Same |
|
| 25 |
+
| Vocabulary rows / tied embeddings | 129,280 / no | Same |
|
| 26 |
+
| Serialized precision | BF16 weights + I64 position IDs | Same |
|
| 27 |
+
|
| 28 |
+
Residual width 32 is independent of the four 32-dimensional attention heads: Q/K/V project to 128 channels and the output projection returns to the residual width. The included runtime exposes these config dimensions while preserving the upstream attention kernels.
|
| 29 |
+
|
| 30 |
+
| File path | Size |
|
| 31 |
+
|------|------|
|
| 32 |
+
| model.safetensors | 18.1MB |
|
| 33 |
+
|
| 34 |
+
### Example usage
|
| 35 |
+
|
| 36 |
+
#### Transformers (✅ verified)
|
| 37 |
+
|
| 38 |
+
Install Transformers 4.52.1, `torchvision`, `Pillow`, `addict`, `easydict` and `einops` and load the included upstream remote code. The source registers only `mha_eager`; its OCR API runs DASH Triton sparse prefill, the reference masked sparse decode, and SDPA in SAM/CLIP.
|
| 39 |
+
|
| 40 |
+
```python
|
| 41 |
+
def run_transformers(model_id: str):
|
| 42 |
+
import torch
|
| 43 |
+
from PIL import Image, ImageDraw
|
| 44 |
+
from transformers import AutoModel, AutoTokenizer
|
| 45 |
+
|
| 46 |
+
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
|
| 47 |
+
model = AutoModel.from_pretrained(model_id, trust_remote_code=True, torch_dtype=torch.bfloat16).eval().cuda()
|
| 48 |
+
page = Image.new("RGB", (800, 1000), "white")
|
| 49 |
+
ImageDraw.Draw(page).text((50, 50), "Hello OCR. Page one.", fill="black")
|
| 50 |
+
text = model.infer(tokenizer, prompt="Read this document.", image_file=page, max_new_tokens=10)
|
| 51 |
+
print(text)
|
| 52 |
+
text, tokens = model.infer_multi(tokenizer, image_files=[page, page.copy()], max_new_tokens=10)
|
| 53 |
+
print(text, tokens)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
if __name__ == "__main__":
|
| 57 |
+
run_transformers(model_id="tiny-random/unlimited-ocr-2")
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
#### SGLang (✅ verified)
|
| 61 |
+
|
| 62 |
+
Install Baidu's pinned `sglang-0.5.20.dev13+g75b9b06b02` wheel from the [source repo](https://huggingface.co/baidu/Unlimited-OCR-2/tree/c14071b549f4fcab10909f3c4acc6a2edb366947/wheel). The included external-model package reads the reduced vision config, separates residual width from attention head dimension, and gathers the TP-sharded DASH index projections before normalization. The upstream attention and routing kernels are retained.
|
| 63 |
+
|
| 64 |
+
```bash
|
| 65 |
+
hf download tiny-random/unlimited-ocr-2 --local-dir ./unlimited-ocr-2
|
| 66 |
+
pip install --no-deps ./unlimited-ocr-2
|
| 67 |
+
|
| 68 |
+
export SGLANG_EXTERNAL_MODEL_PACKAGE=unlimited_ocr_runtime
|
| 69 |
+
export SGLANG_UOCR_EANABLE_DOC_LIMIT=0
|
| 70 |
+
python -m sglang.launch_server \
|
| 71 |
+
--model-path ./unlimited-ocr-2 \
|
| 72 |
+
--tp-size 4 \
|
| 73 |
+
--trust-remote-code \
|
| 74 |
+
--chat-template unlimited-ocr \
|
| 75 |
+
--context-length 4096 \
|
| 76 |
+
--max-total-tokens 8192 \
|
| 77 |
+
--max-running-requests 4 \
|
| 78 |
+
--cuda-graph-backend-decode full \
|
| 79 |
+
--cuda-graph-max-bs-decode 4 \
|
| 80 |
+
--cuda-graph-backend-prefill full \
|
| 81 |
+
--cuda-graph-max-bs-prefill 2048 \
|
| 82 |
+
--cuda-graph-bs-prefill 2048 \
|
| 83 |
+
--max-prefill-tokens 2048 \
|
| 84 |
+
--chunked-prefill-size 2048 \
|
| 85 |
+
--cuda-graph-config '{"prefill": {"backend": "full", "full_prefill_max_req": 4}}' \
|
| 86 |
+
--mem-fraction-static 0.3 \
|
| 87 |
+
--host 0.0.0.0 --port 8000
|
| 88 |
+
```
|
| 89 |
+
|
| 90 |
+
```python
|
| 91 |
+
import base64
|
| 92 |
+
from io import BytesIO
|
| 93 |
+
|
| 94 |
+
import requests
|
| 95 |
+
from PIL import Image, ImageDraw
|
| 96 |
+
|
| 97 |
+
page = Image.new("RGB", (800, 1000), "white")
|
| 98 |
+
ImageDraw.Draw(page).text((50, 50), "Hello OCR", fill="black")
|
| 99 |
+
buffer = BytesIO()
|
| 100 |
+
page.save(buffer, format="PNG")
|
| 101 |
+
image = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
|
| 102 |
+
session = requests.Session()
|
| 103 |
+
session.trust_env = False
|
| 104 |
+
response = session.post("http://127.0.0.1:8000/v1/chat/completions", json={
|
| 105 |
+
"model": "./unlimited-ocr-2",
|
| 106 |
+
"messages": [{"role": "user", "content": [
|
| 107 |
+
{"type": "image_url", "image_url": {"url": image}},
|
| 108 |
+
{"type": "text", "text": "<PAGE><image>Read this document."},
|
| 109 |
+
]}],
|
| 110 |
+
"max_tokens": 10, "temperature": 0,
|
| 111 |
+
}, timeout=120)
|
| 112 |
+
response.raise_for_status()
|
| 113 |
+
print(response.json()["choices"][0]["message"]["content"])
|
| 114 |
+
```
|
| 115 |
+
|
| 116 |
+
### Creating this repo
|
| 117 |
+
|
| 118 |
+
- Code: [.meta/create_model.py](.meta/create_model.py)
|
| 119 |
+
- Model structure: [.meta/model.log](.meta/model.log)
|
| 120 |
+
- Environment: [.meta/env.json](.meta/env.json)
|
| 121 |
+
|
| 122 |
+
### Change log
|
| 123 |
+
|
| 124 |
+
- 2026-10-31: Initial tiny Unlimited-OCR 2 checkpoint with the upstream SR-SWA runtime.
|
| 125 |
+
- 2026-11-01: Separate residual width from the 32-dimensional attention heads to reduce checkpoint size.
|
| 126 |
+
|
| 127 |
+
*This repo is generated by GPT-6.1-Sol.*
|
_sparse_base_impl.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
config.json
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_ring_window": 128,
|
| 3 |
+
"architectures": [
|
| 4 |
+
"UnlimitedOCRForCausalLM"
|
| 5 |
+
],
|
| 6 |
+
"attention_bias": false,
|
| 7 |
+
"attention_dropout": 0.0,
|
| 8 |
+
"auto_map": {
|
| 9 |
+
"AutoConfig": "modeling_unlimitedocr.UnlimitedOCRConfig",
|
| 10 |
+
"AutoModel": "modeling_unlimitedocr.UnlimitedOCRForCausalLM"
|
| 11 |
+
},
|
| 12 |
+
"aux_loss_alpha": 0.001,
|
| 13 |
+
"bos_token_id": 0,
|
| 14 |
+
"candidate_resolutions": [
|
| 15 |
+
[
|
| 16 |
+
1024,
|
| 17 |
+
1024
|
| 18 |
+
]
|
| 19 |
+
],
|
| 20 |
+
"eos_token_id": 1,
|
| 21 |
+
"ep_size": 1,
|
| 22 |
+
"first_k_dense_replace": 1,
|
| 23 |
+
"global_view_pos": "head",
|
| 24 |
+
"hidden_act": "silu",
|
| 25 |
+
"hidden_size": 32,
|
| 26 |
+
"initializer_range": 0.02,
|
| 27 |
+
"intermediate_size": 256,
|
| 28 |
+
"kv_lora_rank": null,
|
| 29 |
+
"language_config": {
|
| 30 |
+
"architectures": [
|
| 31 |
+
"DeepseekOCRForCausalLM"
|
| 32 |
+
],
|
| 33 |
+
"auto_map": {
|
| 34 |
+
"AutoConfig": "configuration_deepseekv2.DeepseekV2Config",
|
| 35 |
+
"AutoModel": "modeling_deepseek.DeepseekV2Model",
|
| 36 |
+
"AutoModelForCausalLM": "modeling_deepseek.DeepseekV2ForCausalLM"
|
| 37 |
+
},
|
| 38 |
+
"bos_token_id": 0,
|
| 39 |
+
"eos_token_id": 1,
|
| 40 |
+
"first_k_dense_replace": 1,
|
| 41 |
+
"hidden_size": 32,
|
| 42 |
+
"intermediate_size": 256,
|
| 43 |
+
"kv_lora_rank": null,
|
| 44 |
+
"lm_head": true,
|
| 45 |
+
"max_position_embeddings": 131072,
|
| 46 |
+
"moe_intermediate_size": 64,
|
| 47 |
+
"n_group": 1,
|
| 48 |
+
"n_routed_experts": 16,
|
| 49 |
+
"n_shared_experts": 2,
|
| 50 |
+
"num_attention_heads": 4,
|
| 51 |
+
"num_experts_per_tok": 6,
|
| 52 |
+
"num_hidden_layers": 4,
|
| 53 |
+
"num_key_value_heads": 4,
|
| 54 |
+
"q_lora_rank": null,
|
| 55 |
+
"qk_nope_head_dim": 0,
|
| 56 |
+
"qk_rope_head_dim": 0,
|
| 57 |
+
"rm_head": false,
|
| 58 |
+
"rope_scaling": {
|
| 59 |
+
"attention_factor": 1.0,
|
| 60 |
+
"beta_fast": 32,
|
| 61 |
+
"beta_slow": 1,
|
| 62 |
+
"factor": 4.0,
|
| 63 |
+
"original_max_position_embeddings": 32768,
|
| 64 |
+
"rope_type": "yarn",
|
| 65 |
+
"type": "yarn"
|
| 66 |
+
},
|
| 67 |
+
"rope_theta": 500000.0,
|
| 68 |
+
"sliding_window_size": 128,
|
| 69 |
+
"topk_group": 1,
|
| 70 |
+
"topk_method": "greedy",
|
| 71 |
+
"torch_dtype": "bfloat16",
|
| 72 |
+
"use_mla": false,
|
| 73 |
+
"v_head_dim": 32,
|
| 74 |
+
"vocab_size": 129280,
|
| 75 |
+
"head_dim": 32
|
| 76 |
+
},
|
| 77 |
+
"lm_head": true,
|
| 78 |
+
"max_position_embeddings": 131072,
|
| 79 |
+
"model_type": "unlimited-ocr",
|
| 80 |
+
"moe_intermediate_size": 64,
|
| 81 |
+
"moe_layer_freq": 1,
|
| 82 |
+
"n_group": 1,
|
| 83 |
+
"n_routed_experts": 16,
|
| 84 |
+
"n_shared_experts": 2,
|
| 85 |
+
"norm_topk_prob": false,
|
| 86 |
+
"num_attention_heads": 4,
|
| 87 |
+
"num_experts_per_tok": 6,
|
| 88 |
+
"num_hidden_layers": 4,
|
| 89 |
+
"num_key_value_heads": 4,
|
| 90 |
+
"pad_token_id": 2,
|
| 91 |
+
"pretraining_tp": 1,
|
| 92 |
+
"projector_config": {
|
| 93 |
+
"input_dim": 64,
|
| 94 |
+
"model_type": "mlp_projector",
|
| 95 |
+
"n_embed": 32,
|
| 96 |
+
"projector_type": "linear"
|
| 97 |
+
},
|
| 98 |
+
"q_lora_rank": null,
|
| 99 |
+
"qk_nope_head_dim": 0,
|
| 100 |
+
"qk_rope_head_dim": 0,
|
| 101 |
+
"rm_head": false,
|
| 102 |
+
"rms_norm_eps": 1e-06,
|
| 103 |
+
"rope_scaling": {
|
| 104 |
+
"attention_factor": 1.0,
|
| 105 |
+
"beta_fast": 32,
|
| 106 |
+
"beta_slow": 1,
|
| 107 |
+
"factor": 4.0,
|
| 108 |
+
"original_max_position_embeddings": 32768,
|
| 109 |
+
"rope_type": "yarn",
|
| 110 |
+
"type": "yarn"
|
| 111 |
+
},
|
| 112 |
+
"rope_theta": 500000.0,
|
| 113 |
+
"routed_scaling_factor": 1.0,
|
| 114 |
+
"scoring_func": "softmax",
|
| 115 |
+
"seq_aux": true,
|
| 116 |
+
"sliding_window": null,
|
| 117 |
+
"sliding_window_size": 128,
|
| 118 |
+
"sparse_attention_config": {
|
| 119 |
+
"dash_floor_per_layer": [
|
| 120 |
+
0.0001,
|
| 121 |
+
0.0001,
|
| 122 |
+
0.0001,
|
| 123 |
+
0.0001
|
| 124 |
+
],
|
| 125 |
+
"dash_image_only": true,
|
| 126 |
+
"dash_prior_floor": 0.0001,
|
| 127 |
+
"dash_query_chunk": 2048,
|
| 128 |
+
"dash_router": "entmax_topk_perlayer",
|
| 129 |
+
"dash_scale_per_layer": [
|
| 130 |
+
128.0,
|
| 131 |
+
128.0,
|
| 132 |
+
128.0,
|
| 133 |
+
128.0
|
| 134 |
+
],
|
| 135 |
+
"dash_scaling_factor": 128.0,
|
| 136 |
+
"dash_sigma": 1.0,
|
| 137 |
+
"dash_stage0": "head_cls",
|
| 138 |
+
"dash_topk_blocks": 12,
|
| 139 |
+
"dash_use_triton": true,
|
| 140 |
+
"image_bucket_constraint": true,
|
| 141 |
+
"image_token_id": 128815,
|
| 142 |
+
"kl_chunk_size": 1024,
|
| 143 |
+
"kl_log_interval": 1,
|
| 144 |
+
"kl_loss_weight": 1.0,
|
| 145 |
+
"page_token_id": 128827,
|
| 146 |
+
"per_page_image_tokens": 273,
|
| 147 |
+
"sparse_attention_freq": [
|
| 148 |
+
1,
|
| 149 |
+
1,
|
| 150 |
+
1,
|
| 151 |
+
1
|
| 152 |
+
],
|
| 153 |
+
"sparse_block_size": 32,
|
| 154 |
+
"sparse_index_dim": 32,
|
| 155 |
+
"sparse_init_block": 1,
|
| 156 |
+
"sparse_local_block": 4,
|
| 157 |
+
"sparse_main_rope_mode": "page2d",
|
| 158 |
+
"sparse_main_rope_tok_freq_mode": "resample",
|
| 159 |
+
"sparse_num_index_heads": 1,
|
| 160 |
+
"sparse_rope_mode": "page2d",
|
| 161 |
+
"sparse_rope_page_mod": 2,
|
| 162 |
+
"sparse_rope_page_pairs": 1,
|
| 163 |
+
"sparse_rope_page_span": 0,
|
| 164 |
+
"sparse_rope_page_theta": 256.0,
|
| 165 |
+
"sparse_rope_theta": 500000.0,
|
| 166 |
+
"sparse_rope_tok_freq_mode": "resample",
|
| 167 |
+
"sparse_rope_tok_theta": 10000.0,
|
| 168 |
+
"sparse_topk_blocks": 16,
|
| 169 |
+
"training_stage": "dash",
|
| 170 |
+
"use_flex_attention": false,
|
| 171 |
+
"use_kl_loss": false,
|
| 172 |
+
"use_sparse_attention": true,
|
| 173 |
+
"use_xformers": false
|
| 174 |
+
},
|
| 175 |
+
"tie_word_embeddings": false,
|
| 176 |
+
"tile_tag": "2D",
|
| 177 |
+
"topk_group": 1,
|
| 178 |
+
"topk_method": "greedy",
|
| 179 |
+
"torch_dtype": "bfloat16",
|
| 180 |
+
"transformers_version": "4.52.1",
|
| 181 |
+
"use_cache": false,
|
| 182 |
+
"use_mla": false,
|
| 183 |
+
"v_head_dim": 32,
|
| 184 |
+
"vision_config": {
|
| 185 |
+
"image_size": 1024,
|
| 186 |
+
"mlp_ratio": 3.7362,
|
| 187 |
+
"model_name": "deeplip_b_l",
|
| 188 |
+
"model_type": "vision",
|
| 189 |
+
"width": {
|
| 190 |
+
"clip-l-14-224": {
|
| 191 |
+
"heads": 4,
|
| 192 |
+
"image_size": 224,
|
| 193 |
+
"layers": 4,
|
| 194 |
+
"patch_size": 14,
|
| 195 |
+
"width": 32
|
| 196 |
+
},
|
| 197 |
+
"sam_vit_b": {
|
| 198 |
+
"downsample_channels": [
|
| 199 |
+
16,
|
| 200 |
+
32
|
| 201 |
+
],
|
| 202 |
+
"global_attn_indexes": [
|
| 203 |
+
2,
|
| 204 |
+
5
|
| 205 |
+
],
|
| 206 |
+
"heads": 4,
|
| 207 |
+
"layers": 6,
|
| 208 |
+
"width": 32,
|
| 209 |
+
"neck_channels": 8
|
| 210 |
+
}
|
| 211 |
+
}
|
| 212 |
+
},
|
| 213 |
+
"vocab_size": 129280,
|
| 214 |
+
"head_dim": 32
|
| 215 |
+
}
|
configuration_deepseek_v2.py
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from transformers.configuration_utils import PretrainedConfig
|
| 2 |
+
from transformers.utils import logging
|
| 3 |
+
|
| 4 |
+
logger = logging.get_logger(__name__)
|
| 5 |
+
|
| 6 |
+
DEEPSEEK_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
|
| 7 |
+
class DeepseekV2Config(PretrainedConfig):
|
| 8 |
+
r"""
|
| 9 |
+
This is the configuration class to store the configuration of a [`DeepseekV2Model`]. It is used to instantiate an DeepSeek
|
| 10 |
+
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
|
| 11 |
+
defaults will yield a similar configuration to that of the DeepSeek-V2 with multi-latent attention.
|
| 12 |
+
|
| 13 |
+
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
| 14 |
+
documentation from [`PretrainedConfig`] for more information.
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
Args:
|
| 18 |
+
vocab_size (`int`, *optional*, defaults to 102400):
|
| 19 |
+
Vocabulary size of the Deep model. Defines the number of different tokens that can be represented by the
|
| 20 |
+
`inputs_ids` passed when calling [`DeepseekV2Model`]
|
| 21 |
+
hidden_size (`int`, *optional*, defaults to 4096):
|
| 22 |
+
Dimension of the hidden representations.
|
| 23 |
+
intermediate_size (`int`, *optional*, defaults to 11008):
|
| 24 |
+
Dimension of the MLP representations.
|
| 25 |
+
moe_intermediate_size (`int`, *optional*, defaults to 1407):
|
| 26 |
+
Dimension of the MoE representations.
|
| 27 |
+
num_hidden_layers (`int`, *optional*, defaults to 32):
|
| 28 |
+
Number of hidden layers in the Transformer decoder.
|
| 29 |
+
num_attention_heads (`int`, *optional*, defaults to 32):
|
| 30 |
+
Number of attention heads for each attention layer in the Transformer decoder.
|
| 31 |
+
n_shared_experts (`int`, *optional*, defaults to None):
|
| 32 |
+
Number of shared experts, None means dense model.
|
| 33 |
+
n_routed_experts (`int`, *optional*, defaults to None):
|
| 34 |
+
Number of routed experts, None means dense model.
|
| 35 |
+
routed_scaling_factor (`float`, *optional*, defaults to 1.0):
|
| 36 |
+
Scaling factor or routed experts.
|
| 37 |
+
topk_method (`str`, *optional*, defaults to `gready`):
|
| 38 |
+
Topk method used in routed gate.
|
| 39 |
+
n_group (`int`, *optional*, defaults to None):
|
| 40 |
+
Number of groups for routed experts.
|
| 41 |
+
topk_group (`int`, *optional*, defaults to None):
|
| 42 |
+
Number of selected groups for each token(for each token, ensuring the selected experts is only within `topk_group` groups).
|
| 43 |
+
num_experts_per_tok (`int`, *optional*, defaults to None):
|
| 44 |
+
Number of selected experts, None means dense model.
|
| 45 |
+
moe_layer_freq (`int`, *optional*, defaults to 1):
|
| 46 |
+
The frequency of the MoE layer: one expert layer for every `moe_layer_freq - 1` dense layers.
|
| 47 |
+
first_k_dense_replace (`int`, *optional*, defaults to 0):
|
| 48 |
+
Number of dense layers in shallow layers(embed->dense->dense->...->dense->moe->moe...->lm_head).
|
| 49 |
+
\--k dense layers--/
|
| 50 |
+
norm_topk_prob (`bool`, *optional*, defaults to False):
|
| 51 |
+
Whether to normalize the weights of the routed experts.
|
| 52 |
+
scoring_func (`str`, *optional*, defaults to 'softmax'):
|
| 53 |
+
Method of computing expert weights.
|
| 54 |
+
aux_loss_alpha (`float`, *optional*, defaults to 0.001):
|
| 55 |
+
Auxiliary loss weight coefficient.
|
| 56 |
+
seq_aux = (`bool`, *optional*, defaults to True):
|
| 57 |
+
Whether to compute the auxiliary loss for each individual sample.
|
| 58 |
+
num_key_value_heads (`int`, *optional*):
|
| 59 |
+
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
| 60 |
+
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
| 61 |
+
`num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
|
| 62 |
+
converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
|
| 63 |
+
by meanpooling all the original heads within that group. For more details checkout [this
|
| 64 |
+
paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
|
| 65 |
+
`num_attention_heads`.
|
| 66 |
+
hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
|
| 67 |
+
The non-linear activation function (function or string) in the decoder.
|
| 68 |
+
max_position_embeddings (`int`, *optional*, defaults to 2048):
|
| 69 |
+
The maximum sequence length that this model might ever be used with.
|
| 70 |
+
initializer_range (`float`, *optional*, defaults to 0.02):
|
| 71 |
+
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
| 72 |
+
rms_norm_eps (`float`, *optional*, defaults to 1e-06):
|
| 73 |
+
The epsilon used by the rms normalization layers.
|
| 74 |
+
use_cache (`bool`, *optional*, defaults to `True`):
|
| 75 |
+
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
| 76 |
+
relevant if `config.is_decoder=True`.
|
| 77 |
+
pad_token_id (`int`, *optional*):
|
| 78 |
+
Padding token id.
|
| 79 |
+
bos_token_id (`int`, *optional*, defaults to 1):
|
| 80 |
+
Beginning of stream token id.
|
| 81 |
+
eos_token_id (`int`, *optional*, defaults to 2):
|
| 82 |
+
End of stream token id.
|
| 83 |
+
pretraining_tp (`int`, *optional*, defaults to 1):
|
| 84 |
+
Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
|
| 85 |
+
document](https://huggingface.co/docs/transformers/parallelism) to understand more about it. This value is
|
| 86 |
+
necessary to ensure exact reproducibility of the pretraining results. Please refer to [this
|
| 87 |
+
issue](https://github.com/pytorch/pytorch/issues/76232).
|
| 88 |
+
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
| 89 |
+
Whether to tie weight embeddings
|
| 90 |
+
rope_theta (`float`, *optional*, defaults to 10000.0):
|
| 91 |
+
The base period of the RoPE embeddings.
|
| 92 |
+
rope_scaling (`Dict`, *optional*):
|
| 93 |
+
Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
|
| 94 |
+
strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
|
| 95 |
+
`{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
|
| 96 |
+
`max_position_embeddings` to the expected new maximum.
|
| 97 |
+
attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
|
| 98 |
+
Whether to use a bias in the query, key, value and output projection layers during self-attention.
|
| 99 |
+
attention_dropout (`float`, *optional*, defaults to 0.0):
|
| 100 |
+
The dropout ratio for the attention probabilities.
|
| 101 |
+
use_mla (`bool`, *optional*, defaults to `True`): Use multi-latent attention or multi-head attention. If True,
|
| 102 |
+
the model will use multi-latent attention, otherwise, it will use multi-head attention.
|
| 103 |
+
|
| 104 |
+
```python
|
| 105 |
+
>>> from transformers import DeepseekV2Model, DeepseekV2Config
|
| 106 |
+
|
| 107 |
+
>>> # Initializing a Deepseek-V2 style configuration
|
| 108 |
+
>>> configuration = DeepseekV2Config()
|
| 109 |
+
|
| 110 |
+
>>> # Accessing the model configuration
|
| 111 |
+
>>> configuration = model.config
|
| 112 |
+
```"""
|
| 113 |
+
|
| 114 |
+
model_type = "deepseek_v2"
|
| 115 |
+
keys_to_ignore_at_inference = ["past_key_values"]
|
| 116 |
+
|
| 117 |
+
def __init__(
|
| 118 |
+
self,
|
| 119 |
+
vocab_size=102400,
|
| 120 |
+
hidden_size=4096,
|
| 121 |
+
intermediate_size=11008,
|
| 122 |
+
moe_intermediate_size = 1407,
|
| 123 |
+
num_hidden_layers=30,
|
| 124 |
+
num_attention_heads=32,
|
| 125 |
+
num_key_value_heads=32,
|
| 126 |
+
n_shared_experts = None,
|
| 127 |
+
n_routed_experts = None,
|
| 128 |
+
ep_size = 1,
|
| 129 |
+
routed_scaling_factor = 1.0,
|
| 130 |
+
kv_lora_rank = 512,
|
| 131 |
+
q_lora_rank = 1536,
|
| 132 |
+
qk_rope_head_dim = 64,
|
| 133 |
+
v_head_dim = 128,
|
| 134 |
+
qk_nope_head_dim = 128,
|
| 135 |
+
topk_method = 'gready',
|
| 136 |
+
n_group = None,
|
| 137 |
+
topk_group = None,
|
| 138 |
+
num_experts_per_tok = None,
|
| 139 |
+
moe_layer_freq = 1,
|
| 140 |
+
first_k_dense_replace = 0,
|
| 141 |
+
norm_topk_prob = False,
|
| 142 |
+
scoring_func = 'softmax',
|
| 143 |
+
aux_loss_alpha = 0.001,
|
| 144 |
+
seq_aux = True,
|
| 145 |
+
hidden_act="silu",
|
| 146 |
+
max_position_embeddings=2048,
|
| 147 |
+
initializer_range=0.02,
|
| 148 |
+
rms_norm_eps=1e-6,
|
| 149 |
+
use_cache=True,
|
| 150 |
+
pad_token_id=None,
|
| 151 |
+
bos_token_id=100000,
|
| 152 |
+
eos_token_id=100001,
|
| 153 |
+
pretraining_tp=1,
|
| 154 |
+
tie_word_embeddings=False,
|
| 155 |
+
rope_theta=10000.0,
|
| 156 |
+
rope_scaling=None,
|
| 157 |
+
attention_bias=False,
|
| 158 |
+
attention_dropout=0.0,
|
| 159 |
+
use_mla=True,
|
| 160 |
+
sliding_window=None,
|
| 161 |
+
**kwargs,
|
| 162 |
+
):
|
| 163 |
+
self.vocab_size = vocab_size
|
| 164 |
+
self.max_position_embeddings = max_position_embeddings
|
| 165 |
+
self.hidden_size = hidden_size
|
| 166 |
+
self.intermediate_size = intermediate_size
|
| 167 |
+
self.moe_intermediate_size = moe_intermediate_size
|
| 168 |
+
self.num_hidden_layers = num_hidden_layers
|
| 169 |
+
self.num_attention_heads = num_attention_heads
|
| 170 |
+
self.n_shared_experts = n_shared_experts
|
| 171 |
+
self.n_routed_experts = n_routed_experts
|
| 172 |
+
self.ep_size = ep_size
|
| 173 |
+
self.routed_scaling_factor = routed_scaling_factor
|
| 174 |
+
self.kv_lora_rank = kv_lora_rank
|
| 175 |
+
self.q_lora_rank = q_lora_rank
|
| 176 |
+
self.qk_rope_head_dim = qk_rope_head_dim
|
| 177 |
+
self.v_head_dim = v_head_dim
|
| 178 |
+
self.qk_nope_head_dim = qk_nope_head_dim
|
| 179 |
+
self.topk_method = topk_method
|
| 180 |
+
self.n_group = n_group
|
| 181 |
+
self.topk_group = topk_group
|
| 182 |
+
self.num_experts_per_tok = num_experts_per_tok
|
| 183 |
+
self.moe_layer_freq = moe_layer_freq
|
| 184 |
+
self.first_k_dense_replace = first_k_dense_replace
|
| 185 |
+
self.norm_topk_prob = norm_topk_prob
|
| 186 |
+
self.scoring_func = scoring_func
|
| 187 |
+
self.aux_loss_alpha = aux_loss_alpha
|
| 188 |
+
self.seq_aux = seq_aux
|
| 189 |
+
# for backward compatibility
|
| 190 |
+
if num_key_value_heads is None:
|
| 191 |
+
num_key_value_heads = num_attention_heads
|
| 192 |
+
|
| 193 |
+
self.num_key_value_heads = num_key_value_heads
|
| 194 |
+
self.hidden_act = hidden_act
|
| 195 |
+
self.initializer_range = initializer_range
|
| 196 |
+
self.rms_norm_eps = float(rms_norm_eps)
|
| 197 |
+
self.pretraining_tp = pretraining_tp
|
| 198 |
+
self.use_cache = use_cache
|
| 199 |
+
self.rope_theta = rope_theta
|
| 200 |
+
self.rope_scaling = rope_scaling
|
| 201 |
+
self.attention_bias = attention_bias
|
| 202 |
+
self.attention_dropout = attention_dropout
|
| 203 |
+
self.use_mla = use_mla
|
| 204 |
+
self.sliding_window = sliding_window
|
| 205 |
+
|
| 206 |
+
super().__init__(
|
| 207 |
+
pad_token_id=pad_token_id,
|
| 208 |
+
bos_token_id=bos_token_id,
|
| 209 |
+
eos_token_id=eos_token_id,
|
| 210 |
+
tie_word_embeddings=tie_word_embeddings,
|
| 211 |
+
**kwargs,
|
| 212 |
+
)
|
conversation.py
ADDED
|
@@ -0,0 +1,280 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
From https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import dataclasses
|
| 6 |
+
from enum import IntEnum, auto
|
| 7 |
+
from typing import Any, Dict, List
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class SeparatorStyle(IntEnum):
|
| 11 |
+
"""Separator styles."""
|
| 12 |
+
|
| 13 |
+
DeepSeek = auto()
|
| 14 |
+
DeepSeekV2 = auto()
|
| 15 |
+
PLAIN = auto()
|
| 16 |
+
ALIGNMENT = auto()
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@dataclasses.dataclass
|
| 20 |
+
class Conversation:
|
| 21 |
+
"""A class that manages prompt templates and keeps all conversation history."""
|
| 22 |
+
|
| 23 |
+
# The name of this template
|
| 24 |
+
name: str
|
| 25 |
+
# The template of the system prompt
|
| 26 |
+
system_template: str = "{system_message}"
|
| 27 |
+
# The system message
|
| 28 |
+
system_message: str = ""
|
| 29 |
+
# The names of two roles
|
| 30 |
+
roles: List[str] = (("USER", "ASSISTANT"),)
|
| 31 |
+
# All messages. Each item is (role, message).
|
| 32 |
+
messages: List[List[str]] = ()
|
| 33 |
+
# The number of few shot examples
|
| 34 |
+
offset: int = 0
|
| 35 |
+
# The separator style and configurations
|
| 36 |
+
sep_style: SeparatorStyle = SeparatorStyle.DeepSeek
|
| 37 |
+
sep: str = "\n"
|
| 38 |
+
sep2: str = None
|
| 39 |
+
# Stop criteria (the default one is EOS token)
|
| 40 |
+
stop_str: str = None
|
| 41 |
+
# Stops generation if meeting any token in this list
|
| 42 |
+
stop_token_ids: List[int] = None
|
| 43 |
+
|
| 44 |
+
def get_prompt(self) -> str:
|
| 45 |
+
"""Get the prompt for generation."""
|
| 46 |
+
system_prompt = self.system_template.format(system_message=self.system_message)
|
| 47 |
+
if self.sep_style == SeparatorStyle.DeepSeek:
|
| 48 |
+
seps = [self.sep, self.sep2]
|
| 49 |
+
if system_prompt == "" or system_prompt is None:
|
| 50 |
+
ret = ""
|
| 51 |
+
else:
|
| 52 |
+
ret = system_prompt + seps[0]
|
| 53 |
+
for i, (role, message) in enumerate(self.messages):
|
| 54 |
+
if message:
|
| 55 |
+
ret += role + ": " + message + seps[i % 2]
|
| 56 |
+
else:
|
| 57 |
+
ret += role + ":"
|
| 58 |
+
return ret
|
| 59 |
+
elif self.sep_style == SeparatorStyle.DeepSeekV2:
|
| 60 |
+
seps = [self.sep, self.sep2]
|
| 61 |
+
if system_prompt == "" or system_prompt is None:
|
| 62 |
+
ret = ""
|
| 63 |
+
else:
|
| 64 |
+
ret = system_prompt + seps[0]
|
| 65 |
+
for i, (role, message) in enumerate(self.messages):
|
| 66 |
+
if message:
|
| 67 |
+
if role == "User":
|
| 68 |
+
ret += "<|sft▁begin|>\n" + message + self.sep #<|sft▁begin|>User Input<|sft▁end|>\nResponse<|end▁of▁sentence|>
|
| 69 |
+
else:
|
| 70 |
+
ret += message + self.sep2
|
| 71 |
+
else:
|
| 72 |
+
ret = ret
|
| 73 |
+
return ret
|
| 74 |
+
|
| 75 |
+
elif self.sep_style == SeparatorStyle.PLAIN:
|
| 76 |
+
seps = [self.sep, self.sep2]
|
| 77 |
+
ret = ""
|
| 78 |
+
for i, (role, message) in enumerate(self.messages):
|
| 79 |
+
if message:
|
| 80 |
+
if type(message) is tuple:
|
| 81 |
+
message, _, _ = message
|
| 82 |
+
if i % 2 == 0:
|
| 83 |
+
ret += message + seps[i % 2]
|
| 84 |
+
else:
|
| 85 |
+
ret += message + seps[i % 2]
|
| 86 |
+
else:
|
| 87 |
+
ret += ""
|
| 88 |
+
return ret
|
| 89 |
+
elif self.sep_style == SeparatorStyle.ALIGNMENT:
|
| 90 |
+
seps = [self.sep, self.sep2]
|
| 91 |
+
ret = ""
|
| 92 |
+
for i, (role, message) in enumerate(self.messages):
|
| 93 |
+
if message:
|
| 94 |
+
if type(message) is tuple:
|
| 95 |
+
message, _, _ = message
|
| 96 |
+
if i % 2 == 0:
|
| 97 |
+
ret += '<image>\n' + seps[i % 2]
|
| 98 |
+
else:
|
| 99 |
+
ret += message + seps[i % 2]
|
| 100 |
+
else:
|
| 101 |
+
ret += ""
|
| 102 |
+
return ret
|
| 103 |
+
else:
|
| 104 |
+
raise ValueError(f"Invalid style: {self.sep_style}")
|
| 105 |
+
|
| 106 |
+
def set_system_message(self, system_message: str):
|
| 107 |
+
"""Set the system message."""
|
| 108 |
+
self.system_message = system_message
|
| 109 |
+
|
| 110 |
+
def append_message(self, role: str, message: str):
|
| 111 |
+
"""Append a new message."""
|
| 112 |
+
self.messages.append([role, message])
|
| 113 |
+
|
| 114 |
+
def update_last_message(self, message: str):
|
| 115 |
+
"""Update the last output.
|
| 116 |
+
|
| 117 |
+
The last message is typically set to be None when constructing the prompt,
|
| 118 |
+
so we need to update it in-place after getting the response from a model.
|
| 119 |
+
"""
|
| 120 |
+
self.messages[-1][1] = message
|
| 121 |
+
|
| 122 |
+
def reset_message(self):
|
| 123 |
+
"""Reset a new message."""
|
| 124 |
+
self.messages = []
|
| 125 |
+
|
| 126 |
+
def to_gradio_chatbot(self):
|
| 127 |
+
"""Convert the conversation to gradio chatbot format."""
|
| 128 |
+
ret = []
|
| 129 |
+
for i, (role, msg) in enumerate(self.messages[self.offset :]):
|
| 130 |
+
if i % 2 == 0:
|
| 131 |
+
ret.append([msg, None])
|
| 132 |
+
else:
|
| 133 |
+
ret[-1][-1] = msg
|
| 134 |
+
return ret
|
| 135 |
+
|
| 136 |
+
def to_openai_api_messages(self):
|
| 137 |
+
"""Convert the conversation to OpenAI chat completion format."""
|
| 138 |
+
system_prompt = self.system_template.format(system_message=self.system_message)
|
| 139 |
+
ret = [{"role": "system", "content": system_prompt}]
|
| 140 |
+
|
| 141 |
+
for i, (_, msg) in enumerate(self.messages[self.offset :]):
|
| 142 |
+
if i % 2 == 0:
|
| 143 |
+
ret.append({"role": "user", "content": msg})
|
| 144 |
+
else:
|
| 145 |
+
if msg is not None:
|
| 146 |
+
ret.append({"role": "assistant", "content": msg})
|
| 147 |
+
return ret
|
| 148 |
+
|
| 149 |
+
def copy(self):
|
| 150 |
+
return Conversation(
|
| 151 |
+
name=self.name,
|
| 152 |
+
system_template=self.system_template,
|
| 153 |
+
system_message=self.system_message,
|
| 154 |
+
roles=self.roles,
|
| 155 |
+
messages=[[x, y] for x, y in self.messages],
|
| 156 |
+
offset=self.offset,
|
| 157 |
+
sep_style=self.sep_style,
|
| 158 |
+
sep=self.sep,
|
| 159 |
+
sep2=self.sep2,
|
| 160 |
+
stop_str=self.stop_str,
|
| 161 |
+
stop_token_ids=self.stop_token_ids,
|
| 162 |
+
)
|
| 163 |
+
|
| 164 |
+
def dict(self):
|
| 165 |
+
return {
|
| 166 |
+
"template_name": self.name,
|
| 167 |
+
"system_message": self.system_message,
|
| 168 |
+
"roles": self.roles,
|
| 169 |
+
"messages": self.messages,
|
| 170 |
+
"offset": self.offset,
|
| 171 |
+
}
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
# A global registry for all conversation templates
|
| 175 |
+
conv_templates: Dict[str, Conversation] = {}
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def register_conv_template(template: Conversation, override: bool = False):
|
| 179 |
+
"""Register a new conversation template."""
|
| 180 |
+
if not override:
|
| 181 |
+
assert template.name not in conv_templates, f"{template.name} has been registered."
|
| 182 |
+
|
| 183 |
+
conv_templates[template.name] = template
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def get_conv_template(name: str) -> Conversation:
|
| 187 |
+
"""Get a conversation template."""
|
| 188 |
+
return conv_templates[name].copy()
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
register_conv_template(
|
| 192 |
+
Conversation(
|
| 193 |
+
name="deepseek",
|
| 194 |
+
system_template="{system_message}",
|
| 195 |
+
# system_message="You are a helpful assistant. Please answer truthfully and write out your "
|
| 196 |
+
# "thinking step by step to be sure you get the right answer.",
|
| 197 |
+
system_message="",
|
| 198 |
+
roles=("<|User|>", "<|Assistant|>"),
|
| 199 |
+
messages=(),
|
| 200 |
+
offset=0,
|
| 201 |
+
sep_style=SeparatorStyle.DeepSeek,
|
| 202 |
+
sep="\n\n",
|
| 203 |
+
sep2="<|end▁of▁sentence|>",
|
| 204 |
+
stop_token_ids=[100001],
|
| 205 |
+
stop_str=["User:", "<|end▁of▁sentence|>"]
|
| 206 |
+
)
|
| 207 |
+
)
|
| 208 |
+
register_conv_template(
|
| 209 |
+
Conversation(
|
| 210 |
+
name="deepseekv2",
|
| 211 |
+
system_template="{system_message}",
|
| 212 |
+
# system_message="You are a helpful assistant. Please answer truthfully and write out your "
|
| 213 |
+
# "thinking step by step to be sure you get the right answer.",
|
| 214 |
+
system_message="",
|
| 215 |
+
roles=("<|User|>", "<|Assistant|>"),
|
| 216 |
+
messages=(),
|
| 217 |
+
offset=0,
|
| 218 |
+
sep_style=SeparatorStyle.DeepSeek,
|
| 219 |
+
sep="",
|
| 220 |
+
sep2="<|end▁of▁sentence|>",
|
| 221 |
+
stop_token_ids=[100001],
|
| 222 |
+
stop_str=["User:", "<|end▁of▁sentence|>"]
|
| 223 |
+
)
|
| 224 |
+
)
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
register_conv_template(
|
| 228 |
+
Conversation(
|
| 229 |
+
name="plain",
|
| 230 |
+
system_template="",
|
| 231 |
+
system_message="",
|
| 232 |
+
roles=("", ""),
|
| 233 |
+
messages=(),
|
| 234 |
+
offset=0,
|
| 235 |
+
sep_style=SeparatorStyle.PLAIN,
|
| 236 |
+
sep="",
|
| 237 |
+
sep2="",
|
| 238 |
+
stop_token_ids=[100001],
|
| 239 |
+
stop_str=['</s>'],
|
| 240 |
+
)
|
| 241 |
+
)
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
register_conv_template(
|
| 245 |
+
Conversation(
|
| 246 |
+
name="alignment",
|
| 247 |
+
system_template="",
|
| 248 |
+
system_message="",
|
| 249 |
+
roles=("", ""),
|
| 250 |
+
messages=(),
|
| 251 |
+
offset=0,
|
| 252 |
+
sep_style=SeparatorStyle.ALIGNMENT,
|
| 253 |
+
sep="",
|
| 254 |
+
sep2="",
|
| 255 |
+
stop_token_ids=[100001],
|
| 256 |
+
stop_str=['</s>'],
|
| 257 |
+
)
|
| 258 |
+
)
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
if __name__ == "__main__":
|
| 262 |
+
print("deepseek template:")
|
| 263 |
+
conv = get_conv_template("deepseek")
|
| 264 |
+
conv.append_message(conv.roles[0], "Hello!")
|
| 265 |
+
conv.append_message(conv.roles[1], "Hi! This is Tony.")
|
| 266 |
+
conv.append_message(conv.roles[0], "Who are you?")
|
| 267 |
+
conv.append_message(conv.roles[1], "I am a helpful assistant.")
|
| 268 |
+
conv.append_message(conv.roles[0], "How are you?")
|
| 269 |
+
conv.append_message(conv.roles[1], None)
|
| 270 |
+
print(conv.get_prompt())
|
| 271 |
+
|
| 272 |
+
print("deepseekv2 template:")
|
| 273 |
+
conv = get_conv_template("deepseekv2")
|
| 274 |
+
conv.append_message(conv.roles[0], "Hello!")
|
| 275 |
+
conv.append_message(conv.roles[1], "Hi! This is Tony.")
|
| 276 |
+
conv.append_message(conv.roles[0], "Who are you?")
|
| 277 |
+
conv.append_message(conv.roles[1], "I am a helpful assistant.")
|
| 278 |
+
conv.append_message(conv.roles[0], "How are you?")
|
| 279 |
+
conv.append_message(conv.roles[1], None)
|
| 280 |
+
print(conv.get_prompt())
|
dash_attention.py
ADDED
|
@@ -0,0 +1,1349 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pure-PyTorch, fully-differentiable DASH attention for Unlimited-OCR.
|
| 2 |
+
|
| 3 |
+
VARIANT B: entmax routing + HARD top-k budget + PER-LAYER entmax temperature
|
| 4 |
+
(see the banner below the imports for the full B-vs-base rationale).
|
| 5 |
+
|
| 6 |
+
DASH = Differentiable and Adaptive Sparse Hierarchical attention
|
| 7 |
+
(https://arxiv.org/abs/2605.18753). Unlike the Lightning-Indexer + top-k DSA
|
| 8 |
+
path of the base implementation (whose block selection is DISCRETE and therefore
|
| 9 |
+
needs a separate warmup + KL-distillation stage to train the indexer), DASH
|
| 10 |
+
routes with **1.5-entmax**, which yields a sparse *and differentiable*
|
| 11 |
+
distribution over key blocks. Gradient reaches the router directly from the LM
|
| 12 |
+
loss, so no KL teacher / no two-stage curriculum is required.
|
| 13 |
+
|
| 14 |
+
This module deliberately uses **plain PyTorch ops only** (no Triton). The only
|
| 15 |
+
custom autograd is the tiny 1.5-entmax Jacobian (a standard, well-known
|
| 16 |
+
closed form over the block axis) -- everything else is ordinary autograd.
|
| 17 |
+
This is the "correctness-first" realization: it is O(S^2) in the attention
|
| 18 |
+
stage (computed in query chunks to bound peak memory), so it is meant for
|
| 19 |
+
training / convergence validation, not for maximum throughput. A fused sparse
|
| 20 |
+
kernel is a later optimization once the recipe is validated.
|
| 21 |
+
|
| 22 |
+
Three stages (mirroring the paper and ``dash-attention-main``):
|
| 23 |
+
stage1 summarize_blocks : one learned key summary per KV block, via a
|
| 24 |
+
softmax attention with a learnable ``head_cls``.
|
| 25 |
+
stage2 route : query->summary scores -> 1.5-entmax over blocks
|
| 26 |
+
(adaptive, differentiable sparse "prior").
|
| 27 |
+
stage3 prior_biased_attn: token-level softmax attention restricted to the
|
| 28 |
+
routed blocks UNION the local window, with
|
| 29 |
+
``log(prior)`` injected as an additive bias so the
|
| 30 |
+
hierarchy stays differentiable end-to-end.
|
| 31 |
+
|
| 32 |
+
Optional image-block restriction: when an image-block mask is supplied, the routing
|
| 33 |
+
candidate set is restricted to image blocks (any page), matching the
|
| 34 |
+
``image_bucket_constraint=False`` semantics of the base implementation.
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
import math
|
| 38 |
+
import os
|
| 39 |
+
|
| 40 |
+
import torch
|
| 41 |
+
import torch.nn as nn
|
| 42 |
+
import torch.nn.functional as F
|
| 43 |
+
|
| 44 |
+
_DASH_DEBUG = os.environ.get("DASH_DEBUG_ANOMALY") == "1"
|
| 45 |
+
|
| 46 |
+
# ===========================================================================
|
| 47 |
+
# VARIANT B -- entmax + hard top-k + per-layer temperature
|
| 48 |
+
# (overlay on patch_config/sparse_dash_code)
|
| 49 |
+
#
|
| 50 |
+
# Differences vs the base DASH overlay:
|
| 51 |
+
# * The attended set is a HARD top-k, k annealed 64 -> 12 (`_KSchedule`), so
|
| 52 |
+
# every query in every layer reads exactly k remote blocks. The base overlay
|
| 53 |
+
# attended the entmax support, whose size varied per layer (13-27 at a mean
|
| 54 |
+
# of 16) and per step.
|
| 55 |
+
# * entmax is KEPT: it still produces the differentiable prior whose
|
| 56 |
+
# mean-centered log is the routing bias, so this stays faithful to the paper's
|
| 57 |
+
# adaptive weighting -- only the cardinality is now fixed.
|
| 58 |
+
# * Each layer runs its OWN closed loop on its OWN entmax temperature, with the
|
| 59 |
+
# current k as setpoint (`_PerLayerScheduler`). Once the count is fixed the
|
| 60 |
+
# temperature's job changes: it has to keep each layer's entmax support ~= k,
|
| 61 |
+
# otherwise the kept blocks either sit at the prior floor (support << k, no
|
| 62 |
+
# gradient) or get an almost flat prior (support >> k, no ranking signal).
|
| 63 |
+
# The base overlay's single global scale could only pin the across-layer mean.
|
| 64 |
+
# * Per-layer one-shot calibration replaces the leader-only calibration, so
|
| 65 |
+
# every layer starts at the initial k instead of inheriting layer 0's
|
| 66 |
+
# temperature.
|
| 67 |
+
# Variant A (patch_config/dash_softmax_topk) is the same budget without entmax.
|
| 68 |
+
# ===========================================================================
|
| 69 |
+
# DASH_LOG_SPARSITY=1 -> print entmax routing sparsity (how many blocks/tokens
|
| 70 |
+
# the router keeps vs drops) during training, throttled to every
|
| 71 |
+
# DASH_LOG_SPARSITY_EVERY calls OF EACH LAYER (so every layer shows up in the log;
|
| 72 |
+
# a single global counter aliased against the layer count and only ever printed
|
| 73 |
+
# the same few layers).
|
| 74 |
+
_LOG_SPARSITY = os.environ.get("DASH_LOG_SPARSITY", "0") == "1"
|
| 75 |
+
_SPARSITY_EVERY = max(1, int(os.environ.get("DASH_LOG_SPARSITY_EVERY", "50")))
|
| 76 |
+
_sparsity_calls = {}
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def _log_train_sparsity(prior, cand, valid, block_size, sid=0):
|
| 80 |
+
"""Print this layer's entmax support vs the hard k budget it is aimed at.
|
| 81 |
+
|
| 82 |
+
``attn`` is what the layer actually reads (== min(k, #candidates), guaranteed
|
| 83 |
+
by the top-k); ``act`` is the entmax support size, the quantity this layer's
|
| 84 |
+
own closed loop drives onto k. act << attn means most kept blocks sit at the
|
| 85 |
+
prior floor and carry no gradient; act >> attn means the prior is flat over
|
| 86 |
+
the kept set. Both are per-layer, hence the L<sid> tag.
|
| 87 |
+
"""
|
| 88 |
+
n = _sparsity_calls.get(sid, 0) + 1
|
| 89 |
+
_sparsity_calls[sid] = n
|
| 90 |
+
# Always print each layer's FIRST call: with EVERY=N the first line of a layer
|
| 91 |
+
# would otherwise take N micro-batches, so a run that dies early (or a short
|
| 92 |
+
# smoke test) showed no per-layer sparsity at all.
|
| 93 |
+
if n != 1 and n % _SPARSITY_EVERY != 0:
|
| 94 |
+
return
|
| 95 |
+
with torch.no_grad():
|
| 96 |
+
nblk = prior.shape[-1]
|
| 97 |
+
c = cand.sum(dim=-1).float() # causal candidate blocks/query
|
| 98 |
+
a = ((prior > 0) & cand).sum(dim=-1).float() # entmax support/query
|
| 99 |
+
t = valid.sum(dim=-1).float() # blocks actually attended
|
| 100 |
+
cm, am, tm = c.mean().item(), a.mean().item(), t.mean().item()
|
| 101 |
+
drop = (1.0 - t.sum() / c.clamp(min=1).sum()).item() * 100.0
|
| 102 |
+
print(f"[DASH sparsity L{sid}] call#{n}: attn {tm:.1f}/{cm:.1f} cand "
|
| 103 |
+
f"blocks/query (k={_KS.k}, drop {drop:.1f}%); entmax act={am:.1f}; "
|
| 104 |
+
f"total_blocks={nblk}; attn ~{tm * block_size:.0f} remote tok/query "
|
| 105 |
+
f"(block_size={block_size}){_SCHED.state_str(sid)}", flush=True)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
# DASH_LOG_NEFF_EVERY>0 -> print, per layer, how many of the k gathered blocks
|
| 109 |
+
# the ATTENTION actually uses. `sparsity` only reports how many blocks the
|
| 110 |
+
# ROUTER selected; it cannot see that the log(prior) bias spans ~9 (floor 1e-4,
|
| 111 |
+
# sigma 1) while qk/sqrt(d) spans ~3-6, so the bias is free to veto a block
|
| 112 |
+
# top-k already paid to read. n_eff = exp(H(w)) with w = each block's share of
|
| 113 |
+
# the remote attention mass: n_eff ~= 2 at k=12 means 10 of the 12 gathered
|
| 114 |
+
# blocks are dead weight -- a failure the temperature loop cannot observe.
|
| 115 |
+
# remote_mass = the remote blocks' share of (remote + local window) mass; n_eff
|
| 116 |
+
# means nothing when the row spends all its mass on the free local context.
|
| 117 |
+
# Cost: one no_grad [R, K*block_size] qk per print. ROWS query rows are sampled
|
| 118 |
+
# and processed CHUNK rows at a time, because these temporaries live alongside
|
| 119 |
+
# the training activations (S=32k is already tight).
|
| 120 |
+
_LOG_NEFF_EVERY = max(0, int(os.environ.get("DASH_LOG_NEFF_EVERY", "0")))
|
| 121 |
+
_LOG_NEFF_ROWS = max(1, int(os.environ.get("DASH_LOG_NEFF_ROWS", "64")))
|
| 122 |
+
_LOG_NEFF_CHUNK = max(1, int(os.environ.get("DASH_LOG_NEFF_CHUNK", "8")))
|
| 123 |
+
_neff_calls = {}
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def _log_neff(q, k, topi, blk_bias, valid, position_ids, block_size,
|
| 127 |
+
local_window, scale, sid=0):
|
| 128 |
+
"""Effective block utilisation of the k blocks this layer gathered.
|
| 129 |
+
|
| 130 |
+
Same throttling convention as _log_train_sparsity: per-layer counter, the
|
| 131 |
+
first call of every layer always prints. Batch element 0 only.
|
| 132 |
+
"""
|
| 133 |
+
n = _neff_calls.get(sid, 0) + 1
|
| 134 |
+
_neff_calls[sid] = n
|
| 135 |
+
if n != 1 and n % _LOG_NEFF_EVERY != 0:
|
| 136 |
+
return
|
| 137 |
+
with torch.no_grad():
|
| 138 |
+
_, H, S, D = q.shape
|
| 139 |
+
K = topi.shape[-1]
|
| 140 |
+
b = 0
|
| 141 |
+
rows = torch.nonzero(valid[b].any(dim=-1)).flatten()
|
| 142 |
+
if rows.numel() == 0:
|
| 143 |
+
print(f"[DASH n_eff L{sid}] call#{n}: no remote selection "
|
| 144 |
+
f"(k={_KS.k})", flush=True)
|
| 145 |
+
return
|
| 146 |
+
if rows.numel() > _LOG_NEFF_ROWS:
|
| 147 |
+
rows = rows[::max(1, rows.numel() // _LOG_NEFF_ROWS)][:_LOG_NEFF_ROWS]
|
| 148 |
+
ar = torch.arange(block_size, device=q.device)
|
| 149 |
+
aw = torch.arange(local_window, device=q.device)
|
| 150 |
+
neff = top1 = rmass = 0.0
|
| 151 |
+
for c0 in range(0, rows.numel(), _LOG_NEFF_CHUNK):
|
| 152 |
+
r = rows[c0:c0 + _LOG_NEFF_CHUNK]
|
| 153 |
+
R = r.numel()
|
| 154 |
+
qpos = position_ids[b, r].long() # [R]
|
| 155 |
+
qi = q[b][:, r, :] # [H,R,D]
|
| 156 |
+
# remote: the exact blocks the kernel/dense path reads, same bias.
|
| 157 |
+
pos = (topi[b][r].clamp_min(0)[:, :, None] * block_size + ar
|
| 158 |
+
).reshape(R, -1) # [R,K*bs]
|
| 159 |
+
keep = ((pos <= qpos[:, None]) & (pos < S)
|
| 160 |
+
& valid[b][r][:, :, None].expand(R, K, block_size
|
| 161 |
+
).reshape(R, -1))
|
| 162 |
+
kk = k[b][:, pos.clamp(0, S - 1).reshape(-1), :].view(
|
| 163 |
+
H, R, K * block_size, D)
|
| 164 |
+
lg = torch.einsum("hrd,hrnd->hrn", qi, kk).float() * scale
|
| 165 |
+
lg = lg + blk_bias[b][r].float()[:, :, None].expand(
|
| 166 |
+
R, K, block_size).reshape(R, -1)
|
| 167 |
+
lg = lg.masked_fill(~keep, float("-inf"))
|
| 168 |
+
# local window: free context the remote blocks compete against.
|
| 169 |
+
lpos = qpos[:, None] - aw # [R,W]
|
| 170 |
+
lkk = k[b][:, lpos.clamp_min(0).reshape(-1), :].view(
|
| 171 |
+
H, R, local_window, D)
|
| 172 |
+
llg = torch.einsum("hrd,hrnd->hrn", qi, lkk).float() * scale
|
| 173 |
+
llg = llg.masked_fill(~(lpos >= 0), float("-inf"))
|
| 174 |
+
# shared max: remote and local must be softmaxed in one denominator.
|
| 175 |
+
m = torch.maximum(lg.amax(-1), llg.amax(-1)).clamp_min(-1e30)
|
| 176 |
+
er = torch.nan_to_num(torch.exp(lg - m[..., None]))
|
| 177 |
+
el = torch.nan_to_num(torch.exp(llg - m[..., None]))
|
| 178 |
+
zr = er.view(H, R, K, block_size).sum(-1) # [H,R,K]
|
| 179 |
+
wr = zr / zr.sum(-1, keepdim=True).clamp_min(1e-20)
|
| 180 |
+
ent = -(wr * torch.log(wr.clamp_min(1e-20))).sum(-1) # [H,R]
|
| 181 |
+
neff += torch.exp(ent).sum().item()
|
| 182 |
+
top1 += wr.amax(-1).sum().item()
|
| 183 |
+
tot = (er.sum(-1) + el.sum(-1)).clamp_min(1e-20)
|
| 184 |
+
rmass += (er.sum(-1) / tot).sum().item()
|
| 185 |
+
cnt = float(H * rows.numel())
|
| 186 |
+
print(f"[DASH n_eff L{sid}] call#{n}: n_eff={neff / cnt:.2f} of K={K} "
|
| 187 |
+
f"(k={_KS.k}); top1_blk={top1 / cnt:.3f}; "
|
| 188 |
+
f"remote_mass={rmass / cnt:.3f}; rows={rows.numel()}", flush=True)
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
# DASH_LOG_RECALL_EVERY>0 -> per layer, the paper-style ORACLE recall of the
|
| 192 |
+
# router's top-k selection (same definition as the DSA report's Fig.3 and the
|
| 193 |
+
# `_log_indexer_recall` of the base implementation):
|
| 194 |
+
#
|
| 195 |
+
# I* = oracle top-k blocks, ranked by the DENSE attention mass P_b that the
|
| 196 |
+
# main branch (q@k) puts on block b.
|
| 197 |
+
# I_hat = the blocks the router actually selected (`topi`, top-k over the prior).
|
| 198 |
+
# block recall@k = |I* ^ I_hat| / k
|
| 199 |
+
# score recall@k = sum_{b in I* ^ I_hat} P_b / sum_{b in I*} P_b
|
| 200 |
+
#
|
| 201 |
+
# Both are measured on the SAME domain the selector uses (candidate blocks only:
|
| 202 |
+
# causal, outside the local window, image-constrained if dash_image_only), so
|
| 203 |
+
# P_b is a softmax normalized over the candidate tokens and mass_all == 1.
|
| 204 |
+
# student_kept / teacher_kept are the same numbers without the intersection:
|
| 205 |
+
# what fraction of the candidate mass the router's k blocks capture vs. what the
|
| 206 |
+
# best possible k blocks capture -- their gap is the pure ranking loss, which is
|
| 207 |
+
# easier to read than either absolute value.
|
| 208 |
+
# Only CONTESTED rows count (n_cand > k); when cand <= k the top-k trivially
|
| 209 |
+
# takes everything and recall is 1.0 by construction, which would inflate the
|
| 210 |
+
# mean. Chance level is ~k/cand_q, so always read recall against the printed
|
| 211 |
+
# cand/q.
|
| 212 |
+
# Cost: one no_grad [H, chunk, S] qk per print (the full dense row, unlike n_eff
|
| 213 |
+
# which only re-reads the k gathered blocks), so keep EVERY large and ROWS small.
|
| 214 |
+
_LOG_RECALL_EVERY = max(0, int(os.environ.get("DASH_LOG_RECALL_EVERY", "0")))
|
| 215 |
+
_LOG_RECALL_ROWS = max(1, int(os.environ.get("DASH_LOG_RECALL_ROWS", "32")))
|
| 216 |
+
_LOG_RECALL_CHUNK = max(1, int(os.environ.get("DASH_LOG_RECALL_CHUNK", "4")))
|
| 217 |
+
_recall_calls = {}
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def _log_oracle_recall(q, k, topi, valid, cand, position_ids, block_size,
|
| 221 |
+
scale, sid=0):
|
| 222 |
+
"""Oracle (dense-attention) recall of this layer's routed block set.
|
| 223 |
+
|
| 224 |
+
Same throttling convention as the other diagnostics (per-layer counter, each
|
| 225 |
+
layer's first call always prints). Batch element 0 only, rank 0 only unless
|
| 226 |
+
DASH_LOG_RECALL_ALL_RANKS=1.
|
| 227 |
+
"""
|
| 228 |
+
n = _recall_calls.get(sid, 0) + 1
|
| 229 |
+
_recall_calls[sid] = n
|
| 230 |
+
if n != 1 and n % _LOG_RECALL_EVERY != 0:
|
| 231 |
+
return
|
| 232 |
+
if os.environ.get("DASH_LOG_RECALL_ALL_RANKS", "0") != "1":
|
| 233 |
+
import torch.distributed as dist
|
| 234 |
+
if dist.is_available() and dist.is_initialized() and dist.get_rank() != 0:
|
| 235 |
+
return
|
| 236 |
+
with torch.no_grad():
|
| 237 |
+
_, H, S, D = q.shape
|
| 238 |
+
K = topi.shape[-1]
|
| 239 |
+
b = 0
|
| 240 |
+
nblk = cand.shape[-1]
|
| 241 |
+
pad = nblk * block_size - S
|
| 242 |
+
# A row is only meaningful if the router had a real choice to make.
|
| 243 |
+
ncand = cand[b].sum(dim=-1) # [S]
|
| 244 |
+
rows = torch.nonzero((ncand > K) & valid[b].any(dim=-1)).flatten()
|
| 245 |
+
if rows.numel() == 0:
|
| 246 |
+
print(f"[DASH recall L{sid}] call#{n}: no contested row "
|
| 247 |
+
f"(k={_KS.k}, K={K})", flush=True)
|
| 248 |
+
return
|
| 249 |
+
if rows.numel() > _LOG_RECALL_ROWS:
|
| 250 |
+
rows = rows[::max(1, rows.numel() // _LOG_RECALL_ROWS)][:_LOG_RECALL_ROWS]
|
| 251 |
+
kpos = torch.arange(S, device=q.device)
|
| 252 |
+
br = sr = kept = teach = t1 = 0.0
|
| 253 |
+
cnt = 0
|
| 254 |
+
for c0 in range(0, rows.numel(), _LOG_RECALL_CHUNK):
|
| 255 |
+
r = rows[c0:c0 + _LOG_RECALL_CHUNK]
|
| 256 |
+
R = r.numel()
|
| 257 |
+
qpos = position_ids[b, r].long() # [R]
|
| 258 |
+
cnd = cand[b][r] # [R,nblk]
|
| 259 |
+
# candidate blocks -> candidate tokens, then re-impose causality
|
| 260 |
+
# (the last candidate block can extend past the query for pad rows).
|
| 261 |
+
keep = cnd.repeat_interleave(block_size, dim=-1)[:, :S] \
|
| 262 |
+
& (kpos[None, :] <= qpos[:, None]) # [R,S]
|
| 263 |
+
lg = torch.einsum("hrd,hsd->hrs", q[b][:, r, :], k[b]).float() * scale
|
| 264 |
+
lg = lg.masked_fill(~keep[None], float("-inf"))
|
| 265 |
+
p = torch.softmax(lg, dim=-1)
|
| 266 |
+
p = torch.nan_to_num(p) # empty rows
|
| 267 |
+
if pad:
|
| 268 |
+
p = F.pad(p, (0, pad), value=0.0)
|
| 269 |
+
# router selection is head-shared, so the oracle is the head-mean
|
| 270 |
+
# block distribution (one teacher per query row).
|
| 271 |
+
p_blk = p.view(H, R, nblk, block_size).sum(-1).mean(0) # [R,nblk]
|
| 272 |
+
p_msk = p_blk.masked_fill(~cnd, -1.0)
|
| 273 |
+
star = p_msk.topk(K, dim=-1).indices
|
| 274 |
+
star_m = torch.zeros_like(cnd).scatter_(-1, star, True) & cnd
|
| 275 |
+
# invalid (padding) selection slots are routed to a scratch column so
|
| 276 |
+
# they cannot be mistaken for a hit on block 0.
|
| 277 |
+
hidx = torch.where(valid[b][r], topi[b][r].clamp_min(0),
|
| 278 |
+
torch.full_like(topi[b][r], nblk))
|
| 279 |
+
hat_m = torch.zeros((R, nblk + 1), dtype=torch.bool, device=q.device
|
| 280 |
+
).scatter_(-1, hidx, True)[:, :nblk] & cnd
|
| 281 |
+
inter = star_m & hat_m
|
| 282 |
+
m_all = (p_blk * cnd).sum(-1).clamp_min(1e-9)
|
| 283 |
+
m_star = (p_blk * star_m).sum(-1)
|
| 284 |
+
ok = m_star > 0
|
| 285 |
+
br += (inter.sum(-1).float() / K)[ok].sum().item()
|
| 286 |
+
sr += ((p_blk * inter).sum(-1) / m_star.clamp_min(1e-9))[ok].sum().item()
|
| 287 |
+
kept += ((p_blk * hat_m).sum(-1) / m_all)[ok].sum().item()
|
| 288 |
+
teach += (m_star / m_all)[ok].sum().item()
|
| 289 |
+
t1 += (p_msk.argmax(-1) == topi[b][r][:, 0])[ok].float().sum().item()
|
| 290 |
+
cnt += int(ok.sum().item())
|
| 291 |
+
if cnt == 0:
|
| 292 |
+
return
|
| 293 |
+
print(f"[DASH recall L{sid}] call#{n}: k={_KS.k} K={K} "
|
| 294 |
+
f"block_recall@k={br / cnt:.4f} score_recall@k={sr / cnt:.4f}; "
|
| 295 |
+
f"student_kept={kept / cnt:.4f} oracle_kept={teach / cnt:.4f} "
|
| 296 |
+
f"top1={t1 / cnt:.4f}; cand/q={ncand[rows].float().mean().item():.1f} "
|
| 297 |
+
f"contested_rows={cnt}/{rows.numel()} nblk={nblk}", flush=True)
|
| 298 |
+
|
| 299 |
+
|
| 300 |
+
# ---------------------------------------------------------------------------
|
| 301 |
+
# Sparsity scheduler -- opt-in via DASH_SPARSITY_SCHED=1 (default OFF, in which
|
| 302 |
+
# case every value below is ignored and behaviour is bit-identical to before).
|
| 303 |
+
#
|
| 304 |
+
# WHY the entmax scale is the sparsity knob: `_route` feeds
|
| 305 |
+
# block_scores * dash_scaling_factor / sqrt(index_dim)
|
| 306 |
+
# to 1.5-entmax, and entmax's support size is governed by the ABSOLUTE scale of
|
| 307 |
+
# its input (scale -> 0 degenerates to uniform/dense, scale up shrinks the
|
| 308 |
+
# support). The shipped 1.0/sqrt(128) ~= 0.088 flattens the score gaps, so the
|
| 309 |
+
# prior stays near-dense. Ramping the scale up = annealing a temperature down.
|
| 310 |
+
#
|
| 311 |
+
# WHY prior_floor must follow: the router's only gradient channel is
|
| 312 |
+
# bias = (log(clamp(p, prior_floor)) - mu) / sigma
|
| 313 |
+
# If the active p_j fall below `prior_floor`, every logp clamps to the same
|
| 314 |
+
# log(floor), the mean-centered bias collapses to 0 and the router stops
|
| 315 |
+
# learning. So the floor is annealed DOWN geometrically as the prior sharpens.
|
| 316 |
+
#
|
| 317 |
+
# Step unit = one training micro-batch forward. Gradient-checkpoint recompute
|
| 318 |
+
# passes are detected (they run inside a backward graph task) and NOT counted,
|
| 319 |
+
# and only the first-registered layer advances the clock, so every DASH layer
|
| 320 |
+
# shares one schedule.
|
| 321 |
+
# ---------------------------------------------------------------------------
|
| 322 |
+
# ---------------------------------------------------------------------------
|
| 323 |
+
# ---------------------------------------------------------------------------
|
| 324 |
+
_LOG_PAGE_EVERY = max(0, int(os.environ.get("DASH_LOG_PAGE_EVERY", "0")))
|
| 325 |
+
_LOG_PAGE_ROWS = max(1, int(os.environ.get("DASH_LOG_PAGE_ROWS", "64")))
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
@torch.no_grad()
|
| 329 |
+
def _log_page_purity(topi, valid, cand, remote, img_page_ids, query_pages,
|
| 330 |
+
block_size, nblk, sid=0, n_call=0):
|
| 331 |
+
if img_page_ids is None or query_pages is None:
|
| 332 |
+
return
|
| 333 |
+
S = topi.shape[1]
|
| 334 |
+
step = max(1, S // _LOG_PAGE_ROWS)
|
| 335 |
+
rows = torch.arange(0, S, step, device=topi.device)[:_LOG_PAGE_ROWS]
|
| 336 |
+
ip = img_page_ids[:1]
|
| 337 |
+
pad = nblk * block_size - ip.shape[-1]
|
| 338 |
+
if pad > 0:
|
| 339 |
+
ip = F.pad(ip, (0, pad), value=-1)
|
| 340 |
+
ip_blk = ip[..., :nblk * block_size].view(nblk, block_size)
|
| 341 |
+
ti, vi = topi[0, rows], valid[0, rows] # [R,K]
|
| 342 |
+
qp = query_pages[0, rows][:, None, None] # [R,1,1]
|
| 343 |
+
bp = ip_blk[ti] # [R,K,block_size]
|
| 344 |
+
is_img = bp >= 0
|
| 345 |
+
n = int(vi.sum().clamp(min=1))
|
| 346 |
+
nonimg = int(((is_img.sum(-1) == 0) & vi).sum())
|
| 347 |
+
other = (is_img & (bp != qp)).sum(-1)
|
| 348 |
+
same = ((bp == qp) & (qp >= 0)).sum(-1)
|
| 349 |
+
cross = int(((other > 0) & vi).sum())
|
| 350 |
+
pure = int(((other > 0) & (same == 0) & vi).sum())
|
| 351 |
+
empty = float((~(cand[0, rows] & remote[0, rows]).any(-1)).float().mean() * 100)
|
| 352 |
+
no_page = float((query_pages[0, rows] < 0).float().mean() * 100)
|
| 353 |
+
print(f"[DASH page L{sid}] call#{n_call}: rows={len(rows)} sel={n} "
|
| 354 |
+
f"non_image={100 * nonimg / n:.2f}% cross_block={100 * cross / n:.2f}% "
|
| 355 |
+
f"pure_cross={100 * pure / n:.2f}% cand_empty={empty:.1f}% "
|
| 356 |
+
f"no_page={no_page:.1f}%", flush=True)
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
_SCHED_ON = os.environ.get("DASH_SPARSITY_SCHED", "0") == "1"
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
def _envf(name, default):
|
| 363 |
+
v = os.environ.get(name)
|
| 364 |
+
return default if v is None or v == "" else float(v)
|
| 365 |
+
|
| 366 |
+
|
| 367 |
+
def _envi(name, default):
|
| 368 |
+
v = os.environ.get(name)
|
| 369 |
+
return default if v is None or v == "" else int(v)
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
# ---------------------------------------------------------------------------
|
| 373 |
+
# Hard top-k budget with a dense-to-sparse k ANNEAL.
|
| 374 |
+
#
|
| 375 |
+
# The routed-block count is a HARD, deterministic budget: every query attends
|
| 376 |
+
# min(k, #candidates) remote blocks in EVERY layer, so "same number of active
|
| 377 |
+
# blocks in every layer at every step" holds by construction. entmax still runs
|
| 378 |
+
# (it produces the differentiable weights on the kept blocks), and each layer's
|
| 379 |
+
# entmax temperature is closed-looped onto this same k -- see
|
| 380 |
+
# `_PerLayerScheduler` for why that matters once the count is fixed.
|
| 381 |
+
#
|
| 382 |
+
# k anneals geometrically K_START -> K_END over K_ANNEAL_STEPS optimizer steps.
|
| 383 |
+
# WHY anneal at all when the budget is exact: the base checkpoint was pretrained
|
| 384 |
+
# dense, and 12 blocks is ~2.3% of the KV at 16k. Truncating that hard at step 0
|
| 385 |
+
# with lr=1e-4 full-finetune and no warmup perturbs the representation before the
|
| 386 |
+
# router has learned to rank blocks; starting wide gives the router a usable
|
| 387 |
+
# gradient signal while the model is still close to its pretrained state.
|
| 388 |
+
#
|
| 389 |
+
# k MUST be constant for a whole micro-batch: with gradient checkpointing the
|
| 390 |
+
# layers are recomputed in backward, and a k that moved mid-forward changes the
|
| 391 |
+
# gathered width -> `CheckpointError: recomputed values have different metadata`.
|
| 392 |
+
# So it is refreshed ONLY at a micro-batch edge, by the leader layer.
|
| 393 |
+
# ---------------------------------------------------------------------------
|
| 394 |
+
class _KSchedule:
|
| 395 |
+
# gc scans are not free, so only probe for the TrainerState a few times.
|
| 396 |
+
_PROBE_AT = (1, 4, 16, 64, 256)
|
| 397 |
+
|
| 398 |
+
def __init__(self):
|
| 399 |
+
self.k_start = _envi("DASH_K_START", 64)
|
| 400 |
+
self.k_end = _envi("DASH_K_END", 12)
|
| 401 |
+
self.steps = max(1, _envi("DASH_K_ANNEAL_STEPS", 1000))
|
| 402 |
+
self.log_every = max(1, _envi("DASH_K_LOG_EVERY", 50))
|
| 403 |
+
self._k = self.k_start
|
| 404 |
+
self._micro = 0
|
| 405 |
+
self._step = 0
|
| 406 |
+
self._state = None
|
| 407 |
+
self._leader = None
|
| 408 |
+
self._n_reg = 0
|
| 409 |
+
self._logged = False
|
| 410 |
+
|
| 411 |
+
def register(self):
|
| 412 |
+
"""Returns an id; the first caller becomes the clock leader."""
|
| 413 |
+
sid = self._n_reg
|
| 414 |
+
self._n_reg += 1
|
| 415 |
+
if self._leader is None:
|
| 416 |
+
self._leader = sid
|
| 417 |
+
return sid
|
| 418 |
+
|
| 419 |
+
def _probe_state(self):
|
| 420 |
+
"""Locate the live HF TrainerState (gives the optimizer-step clock)."""
|
| 421 |
+
try:
|
| 422 |
+
import gc
|
| 423 |
+
for o in gc.get_objects():
|
| 424 |
+
if (type(o).__name__ == "TrainerState"
|
| 425 |
+
and int(getattr(o, "max_steps", 0) or 0) > 0):
|
| 426 |
+
return o
|
| 427 |
+
except Exception:
|
| 428 |
+
pass
|
| 429 |
+
return None
|
| 430 |
+
|
| 431 |
+
def k_for(self, step):
|
| 432 |
+
"""Geometric interpolation K_START -> K_END, clamped to the endpoints."""
|
| 433 |
+
p = min(1.0, max(0.0, step / float(self.steps)))
|
| 434 |
+
k = self.k_start * (self.k_end / float(self.k_start)) ** p
|
| 435 |
+
lo, hi = min(self.k_start, self.k_end), max(self.k_start, self.k_end)
|
| 436 |
+
return max(lo, min(hi, int(round(k))))
|
| 437 |
+
|
| 438 |
+
def advance(self, sid, training):
|
| 439 |
+
"""Refresh k once per micro-batch; skip gradient-checkpoint recompute."""
|
| 440 |
+
if not training or sid != self._leader:
|
| 441 |
+
return
|
| 442 |
+
if torch._C._current_graph_task_id() != -1: # inside backward
|
| 443 |
+
return
|
| 444 |
+
self._micro += 1
|
| 445 |
+
if self._state is None and self._micro in self._PROBE_AT:
|
| 446 |
+
self._state = self._probe_state()
|
| 447 |
+
self._step = (int(self._state.global_step) if self._state is not None
|
| 448 |
+
else self._micro)
|
| 449 |
+
self._k = self.k_for(self._step)
|
| 450 |
+
if not self._logged:
|
| 451 |
+
self._logged = True
|
| 452 |
+
print(f"[DASH k-sched] hard top-k: {self.k_start} -> {self.k_end} "
|
| 453 |
+
f"geometric over {self.steps} steps "
|
| 454 |
+
f"(clock={'TrainerState' if self._state is not None else 'micro-batch'})",
|
| 455 |
+
flush=True)
|
| 456 |
+
elif self._micro % self.log_every == 0:
|
| 457 |
+
print(f"[DASH k-sched] step={self._step} k={self._k}", flush=True)
|
| 458 |
+
|
| 459 |
+
@property
|
| 460 |
+
def k(self):
|
| 461 |
+
return self._k
|
| 462 |
+
|
| 463 |
+
|
| 464 |
+
_KS = _KSchedule()
|
| 465 |
+
|
| 466 |
+
|
| 467 |
+
class _PerLayerScheduler:
|
| 468 |
+
"""Per-layer entmax temperature, closed-looped onto the CURRENT k budget.
|
| 469 |
+
|
| 470 |
+
Variant B keeps entmax (adaptive, differentiable sparse prior) but the
|
| 471 |
+
attended set is a hard top-k, so entmax no longer decides HOW MANY blocks are
|
| 472 |
+
read -- it decides how the kept blocks are WEIGHTED, and therefore how much
|
| 473 |
+
gradient each one carries. Two failure modes follow, both per-layer:
|
| 474 |
+
|
| 475 |
+
* support << k -> most of the k kept blocks have prior 0, their bias clamps
|
| 476 |
+
to log(prior_floor) and they contribute no gradient.
|
| 477 |
+
* support >> k -> the prior is flat over the kept blocks, so the bias
|
| 478 |
+
carries almost no ranking information.
|
| 479 |
+
|
| 480 |
+
So each layer runs its OWN loop driving its entmax support size onto the
|
| 481 |
+
current k (``_KS.k``, which itself anneals 64->12). The base overlay shared
|
| 482 |
+
ONE global scale and could only pin the across-layer MEAN: the per-layer
|
| 483 |
+
counts spread 13-27 at a mean of 16, i.e. some layers saturated while others
|
| 484 |
+
stayed flat. ``prior_floor`` follows each layer's own scale geometrically.
|
| 485 |
+
|
| 486 |
+
The scale MUST be constant for a whole micro-batch: with gradient
|
| 487 |
+
checkpointing the layers are recomputed in backward, and a scale that moved
|
| 488 |
+
mid-forward changes the entmax support -> a different bias -> the recompute
|
| 489 |
+
disagrees with the saved forward. So every layer refreshes its scale at most
|
| 490 |
+
once per micro-batch, from the PREVIOUS micro-batch's measurement.
|
| 491 |
+
"""
|
| 492 |
+
|
| 493 |
+
# gc scans are not free, so only probe for the TrainerState a few times.
|
| 494 |
+
_PROBE_AT = (1, 4, 16, 64, 256)
|
| 495 |
+
|
| 496 |
+
def __init__(self):
|
| 497 |
+
self.on = os.environ.get("DASH_SPARSITY_SCHED", "0") == "1"
|
| 498 |
+
self.scale_max = _envf("DASH_SCHED_SCALE_MAX", 128.0)
|
| 499 |
+
self.scale_min = _envf("DASH_SCHED_SCALE_MIN", 0.05)
|
| 500 |
+
self.log_every = max(1, _envi("DASH_SCHED_LOG_EVERY", 50))
|
| 501 |
+
self.gain = _envf("DASH_SCHED_GAIN", 0.1) # log-domain P gain
|
| 502 |
+
self.ema = _envf("DASH_SCHED_EMA", 0.9) # measurement smoothing
|
| 503 |
+
self.max_step_ratio = _envf("DASH_SCHED_MAX_STEP", 1.05) # per-step cap
|
| 504 |
+
# one-shot PER-LAYER calibration: bisect this layer's scale on its first
|
| 505 |
+
# real batch so it starts exactly at the initial k, instead of inheriting
|
| 506 |
+
# a temperature tuned for another layer's score spread.
|
| 507 |
+
self.calibrate_on = _envi("DASH_SCHED_CALIB", 1) == 1
|
| 508 |
+
self.calib_rows = max(16, _envi("DASH_SCHED_CALIB_ROWS", 512))
|
| 509 |
+
self.floor_start = None # resolved from config on first register()
|
| 510 |
+
self.floor_end = _envf("DASH_SCHED_FLOOR_END", 1.0e-4)
|
| 511 |
+
self.scale_init = None
|
| 512 |
+
self.scale_ref = None # floor interpolates over [init, ref]
|
| 513 |
+
self._scale = {} # sid -> entmax scale
|
| 514 |
+
self._floor = {} # sid -> prior floor
|
| 515 |
+
self._measured = {} # sid -> EMA of entmax-active blocks/query
|
| 516 |
+
self._pending = {} # sid -> newest raw measurement
|
| 517 |
+
self._micro = {} # sid -> own micro-batch counter
|
| 518 |
+
self._calibrated = set()
|
| 519 |
+
self._step = 0
|
| 520 |
+
self._state = None
|
| 521 |
+
self._leader = None
|
| 522 |
+
self._n_reg = 0
|
| 523 |
+
self._logged = False
|
| 524 |
+
# -- setup ---------------------------------------------------------------
|
| 525 |
+
def register(self, cfg_scale, cfg_floor):
|
| 526 |
+
"""Returns an id; the first caller becomes the clock leader."""
|
| 527 |
+
if self.scale_init is None:
|
| 528 |
+
self.scale_init = _envf("DASH_SCHED_SCALE_START", cfg_scale)
|
| 529 |
+
self.scale_ref = _envf("DASH_SCHED_SCALE_END", 8.0)
|
| 530 |
+
self.floor_start = _envf("DASH_SCHED_FLOOR_START", cfg_floor)
|
| 531 |
+
sid = self._n_reg
|
| 532 |
+
self._n_reg += 1
|
| 533 |
+
if self._leader is None:
|
| 534 |
+
self._leader = sid
|
| 535 |
+
self._scale[sid] = self.scale_init
|
| 536 |
+
self._floor[sid] = self.floor_start
|
| 537 |
+
self._micro[sid] = 0
|
| 538 |
+
return sid
|
| 539 |
+
|
| 540 |
+
# -- clock ---------------------------------------------------------------
|
| 541 |
+
def _probe_state(self):
|
| 542 |
+
"""Locate the live HF TrainerState (gives the optimizer-step clock)."""
|
| 543 |
+
try:
|
| 544 |
+
import gc
|
| 545 |
+
for o in gc.get_objects():
|
| 546 |
+
if (type(o).__name__ == "TrainerState"
|
| 547 |
+
and int(getattr(o, "max_steps", 0) or 0) > 0):
|
| 548 |
+
return o
|
| 549 |
+
except Exception:
|
| 550 |
+
pass
|
| 551 |
+
return None
|
| 552 |
+
|
| 553 |
+
def advance(self, sid, training):
|
| 554 |
+
"""Refresh THIS layer's scale; skip gradient-checkpoint recompute."""
|
| 555 |
+
if not self.on or not training:
|
| 556 |
+
return
|
| 557 |
+
if torch._C._current_graph_task_id() != -1: # inside backward
|
| 558 |
+
return
|
| 559 |
+
n = self._micro.get(sid, 0) + 1
|
| 560 |
+
self._micro[sid] = n
|
| 561 |
+
if sid == self._leader:
|
| 562 |
+
if self._state is None and n in self._PROBE_AT:
|
| 563 |
+
self._state = self._probe_state()
|
| 564 |
+
self._step = (int(self._state.global_step) if self._state is not None
|
| 565 |
+
else n)
|
| 566 |
+
# fold the previous micro-batch's measurement in, then P-control
|
| 567 |
+
m = self._pending.pop(sid, None)
|
| 568 |
+
if m is not None:
|
| 569 |
+
prev = self._measured.get(sid)
|
| 570 |
+
self._measured[sid] = m if prev is None else (
|
| 571 |
+
self.ema * prev + (1.0 - self.ema) * m)
|
| 572 |
+
self._scale[sid] = self._closed_loop_scale(sid)
|
| 573 |
+
self._floor[sid] = self._floor_for(self._scale[sid])
|
| 574 |
+
if not self._logged:
|
| 575 |
+
self._logged = True
|
| 576 |
+
print(f"[DASH sched] per-layer closed loop ON: setpoint = current k "
|
| 577 |
+
f"(anneal {_KS.k_start}->{_KS.k_end}), scale init "
|
| 578 |
+
f"{self.scale_init:g} in [{self.scale_min:g}, {self.scale_max:g}], "
|
| 579 |
+
f"floor {self.floor_start:g}->{self.floor_end:g}, gain={self.gain:g} "
|
| 580 |
+
f"ema={self.ema:g} max_step={self.max_step_ratio:g}", flush=True)
|
| 581 |
+
elif sid == self._leader and n % self.log_every == 0:
|
| 582 |
+
print(f"[DASH sched] step={self._step} k={_KS.k}" + self._table(),
|
| 583 |
+
flush=True)
|
| 584 |
+
|
| 585 |
+
def _table(self):
|
| 586 |
+
"""`| L0 scale=.. act=.. | L1 ...` -- every layer's temperature + support."""
|
| 587 |
+
out = []
|
| 588 |
+
for s in sorted(self._scale):
|
| 589 |
+
act = self._measured.get(s)
|
| 590 |
+
out.append(f" | L{s} scale={self._scale[s]:.2f} "
|
| 591 |
+
f"act={'-' if act is None else format(act, '.1f')}")
|
| 592 |
+
return "".join(out)
|
| 593 |
+
# -- closed loop ---------------------------------------------------------
|
| 594 |
+
def _floor_for(self, scale):
|
| 595 |
+
"""Tie the floor to how far the scale has travelled (log domain).
|
| 596 |
+
|
| 597 |
+
WHY the floor must follow: the router's gradient channel is
|
| 598 |
+
``bias = (log(clamp(p, floor)) - mu)/sigma``. If the active p_j fall below
|
| 599 |
+
`floor`, every logp clamps to the same value, the mean-centered bias
|
| 600 |
+
collapses to 0 and the router stops learning. A sharper prior has smaller
|
| 601 |
+
active p, so the floor is annealed DOWN with the scale.
|
| 602 |
+
"""
|
| 603 |
+
a, b = self.scale_init, self.scale_ref
|
| 604 |
+
if not a or b <= a:
|
| 605 |
+
p = 0.0
|
| 606 |
+
else:
|
| 607 |
+
p = min(1.0, max(0.0, math.log(max(scale, 1e-8) / a) / math.log(b / a)))
|
| 608 |
+
fa, fb = self.floor_start, self.floor_end
|
| 609 |
+
return fa * (fb / max(fa, 1e-12)) ** p # geometric, floor goes DOWN
|
| 610 |
+
|
| 611 |
+
def _closed_loop_scale(self, sid):
|
| 612 |
+
"""Multiplicative P-control in the log domain, per layer: support above
|
| 613 |
+
the k budget -> raise the scale (sharper entmax), below -> back off."""
|
| 614 |
+
tgt = float(_KS.k)
|
| 615 |
+
m = self._measured.get(sid)
|
| 616 |
+
cur = self._scale[sid]
|
| 617 |
+
if tgt <= 0 or m is None:
|
| 618 |
+
return cur
|
| 619 |
+
ratio = math.exp(self.gain * math.log(max(m, 1e-6) / tgt))
|
| 620 |
+
ratio = min(self.max_step_ratio, max(1.0 / self.max_step_ratio, ratio))
|
| 621 |
+
return min(self.scale_max, max(self.scale_min, cur * ratio))
|
| 622 |
+
|
| 623 |
+
def observe(self, sid, prior, cand, training):
|
| 624 |
+
"""Record this layer's entmax support size for its own loop.
|
| 625 |
+
|
| 626 |
+
MEASUREMENT ONLY -- the scale it feeds is applied by ``advance()`` at the
|
| 627 |
+
next micro-batch boundary (mutating it mid-forward breaks the
|
| 628 |
+
gradient-checkpoint recompute).
|
| 629 |
+
"""
|
| 630 |
+
if not self.on or not training:
|
| 631 |
+
return
|
| 632 |
+
if torch._C._current_graph_task_id() != -1:
|
| 633 |
+
return
|
| 634 |
+
with torch.no_grad():
|
| 635 |
+
a = ((prior > 0) & cand).sum(dim=-1).float().mean()
|
| 636 |
+
try:
|
| 637 |
+
import torch.distributed as dist
|
| 638 |
+
if dist.is_available() and dist.is_initialized():
|
| 639 |
+
dist.all_reduce(a, op=dist.ReduceOp.AVG) # keep ranks in sync
|
| 640 |
+
except Exception:
|
| 641 |
+
pass
|
| 642 |
+
m = float(a.item())
|
| 643 |
+
if math.isfinite(m):
|
| 644 |
+
self._pending[sid] = m
|
| 645 |
+
|
| 646 |
+
def needs_calib(self, sid, training):
|
| 647 |
+
return (self.on and self.calibrate_on and sid not in self._calibrated
|
| 648 |
+
and training and torch._C._current_graph_task_id() == -1)
|
| 649 |
+
|
| 650 |
+
@torch.no_grad()
|
| 651 |
+
def calibrate(self, sid, block_scores, cand, neg, sqrt_d):
|
| 652 |
+
"""Bisect THIS layer's entmax scale on its first batch so its support
|
| 653 |
+
starts at the current k budget.
|
| 654 |
+
|
| 655 |
+
The active-block count is monotonically non-increasing in the scale, so a
|
| 656 |
+
geometric bisection converges in a few iterations. Runs on a strided
|
| 657 |
+
subset of the query rows (the full [B,S,nblk] would sort tens of millions
|
| 658 |
+
of entries per probe).
|
| 659 |
+
"""
|
| 660 |
+
self._calibrated.add(sid) # never retry, even if this bails out
|
| 661 |
+
target = float(_KS.k)
|
| 662 |
+
stride = max(1, block_scores.shape[1] // self.calib_rows)
|
| 663 |
+
bs = block_scores[:, ::stride].float()
|
| 664 |
+
cd = cand[:, ::stride]
|
| 665 |
+
|
| 666 |
+
def active(s):
|
| 667 |
+
x = torch.where(cd, bs * (s / sqrt_d), bs.new_full((), neg))
|
| 668 |
+
return float(((entmax15(x, dim=-1) > 0) & cd).sum(dim=-1).float().mean())
|
| 669 |
+
|
| 670 |
+
lo, hi = self.scale_min, self.scale_max
|
| 671 |
+
if active(lo) <= target:
|
| 672 |
+
s, why = lo, f"already at {active(lo):.1f} blk <= k"
|
| 673 |
+
elif active(hi) > target:
|
| 674 |
+
s, why = hi, f"SCALE_MAX only reaches {active(hi):.1f} blk"
|
| 675 |
+
else:
|
| 676 |
+
for _ in range(20):
|
| 677 |
+
mid = math.sqrt(lo * hi)
|
| 678 |
+
if active(mid) > target:
|
| 679 |
+
lo = mid
|
| 680 |
+
else:
|
| 681 |
+
hi = mid
|
| 682 |
+
s, why = hi, f"bisected, {active(hi):.1f} blk"
|
| 683 |
+
try:
|
| 684 |
+
import torch.distributed as dist
|
| 685 |
+
if dist.is_available() and dist.is_initialized():
|
| 686 |
+
t = torch.tensor([s], device=block_scores.device, dtype=torch.float32)
|
| 687 |
+
dist.all_reduce(t, op=dist.ReduceOp.AVG) # one scale per layer, all ranks
|
| 688 |
+
s = float(t.item())
|
| 689 |
+
except Exception:
|
| 690 |
+
pass
|
| 691 |
+
self._scale[sid] = s
|
| 692 |
+
self._floor[sid] = self._floor_for(s)
|
| 693 |
+
print(f"[DASH sched] L{sid} calibrated scale={s:.3f} for k={target:g} "
|
| 694 |
+
f"({why})", flush=True)
|
| 695 |
+
|
| 696 |
+
# -- readout -------------------------------------------------------------
|
| 697 |
+
def scale(self, sid, fallback):
|
| 698 |
+
return self._scale.get(sid, fallback) if self.on else fallback
|
| 699 |
+
|
| 700 |
+
def floor(self, sid, fallback):
|
| 701 |
+
return self._floor.get(sid, fallback) if self.on else fallback
|
| 702 |
+
|
| 703 |
+
def state_str(self, sid):
|
| 704 |
+
if not self.on or sid not in self._scale:
|
| 705 |
+
return ""
|
| 706 |
+
return (f" [sched step={self._step} scale={self._scale[sid]:.3f} "
|
| 707 |
+
f"floor={self._floor[sid]:.2e}]")
|
| 708 |
+
|
| 709 |
+
|
| 710 |
+
_SCHED = _PerLayerScheduler()
|
| 711 |
+
|
| 712 |
+
|
| 713 |
+
def _dbg_finite(name, t):
|
| 714 |
+
"""Forward-side finiteness probe (only under DASH_DEBUG_ANOMALY=1)."""
|
| 715 |
+
if _DASH_DEBUG and torch.is_tensor(t) and not torch.isfinite(t).all():
|
| 716 |
+
print(f"[DASH-DBG fwd] non-finite in {name}: shape={tuple(t.shape)} "
|
| 717 |
+
f"nan={int(torch.isnan(t).sum())} inf={int(torch.isinf(t).sum())}",
|
| 718 |
+
flush=True)
|
| 719 |
+
|
| 720 |
+
|
| 721 |
+
def _dbg_grad(name, t):
|
| 722 |
+
"""Register a backward hook that reports the first non-finite gradient."""
|
| 723 |
+
if _DASH_DEBUG and torch.is_tensor(t) and t.requires_grad:
|
| 724 |
+
def _hook(g, _n=name):
|
| 725 |
+
if g is not None and not torch.isfinite(g).all():
|
| 726 |
+
print(f"[DASH-DBG bwd] non-finite GRAD at {_n}: shape={tuple(g.shape)} "
|
| 727 |
+
f"nan={int(torch.isnan(g).sum())} inf={int(torch.isinf(g).sum())}",
|
| 728 |
+
flush=True)
|
| 729 |
+
return None
|
| 730 |
+
t.register_hook(_hook)
|
| 731 |
+
|
| 732 |
+
# log-floor used for zeroed entmax priors; `base > _LOG_TINY + 1` <=> prior > 0.
|
| 733 |
+
_LOG_TINY = math.log(1e-20)
|
| 734 |
+
# finite "-inf" for SDPA additive masks (real -inf makes SDPA's fused backward
|
| 735 |
+
# produce NaN gradients via 0*inf). exp(_MASK_NEG - rowmax) underflows to 0.
|
| 736 |
+
_MASK_NEG = -1.0e9
|
| 737 |
+
# The router NEVER uses real -inf: masked_fill(-inf) + amax makes fully-masked
|
| 738 |
+
# rows -inf, and that -inf reaches the amax/matmul backward, which manufactures
|
| 739 |
+
# inf/NaN grads (0*inf) in q_norm/k_norm -> the MulBackward0 NaN anomaly.
|
| 740 |
+
# A finite sentinel keeps every backward well defined.
|
| 741 |
+
_SCORE_NEG = -1.0e9
|
| 742 |
+
# fp32 overflow guard: a *finite but huge* grad (e.g. 1e37) still overflows the
|
| 743 |
+
# grad_weight reduction inside RMSNorm's mul backward (sum -> +-inf -> NaN), so
|
| 744 |
+
# grads must be BOUNDED, not merely nan_to_num'd.
|
| 745 |
+
_GRAD_CLAMP = 1.0e4
|
| 746 |
+
_ACT_CLAMP = 1.0e4
|
| 747 |
+
|
| 748 |
+
|
| 749 |
+
def _sanitize_grad(g):
|
| 750 |
+
"""nan_to_num + bound: keeps every downstream reduction inside fp32 range."""
|
| 751 |
+
if g is None:
|
| 752 |
+
return None
|
| 753 |
+
return torch.nan_to_num(g, nan=0.0, posinf=0.0, neginf=0.0).clamp(
|
| 754 |
+
-_GRAD_CLAMP, _GRAD_CLAMP)
|
| 755 |
+
|
| 756 |
+
|
| 757 |
+
# ---------------------------------------------------------------------------
|
| 758 |
+
# 1.5-entmax (Peters, Niculae & Martins, 2019). Exact forward + backward.
|
| 759 |
+
# The ONLY custom autograd in this file. Operates over `dim` (the block axis).
|
| 760 |
+
# Masked-out entries must be passed as a finite, very-negative value (not -inf)
|
| 761 |
+
# so the sorting-based threshold stays numerically well-defined; they receive
|
| 762 |
+
# probability exactly 0.
|
| 763 |
+
# ---------------------------------------------------------------------------
|
| 764 |
+
def _make_ix_like(X, dim):
|
| 765 |
+
d = X.size(dim)
|
| 766 |
+
rho = torch.arange(1, d + 1, device=X.device, dtype=X.dtype)
|
| 767 |
+
view = [1] * X.dim()
|
| 768 |
+
view[0] = -1
|
| 769 |
+
return rho.view(view).transpose(0, dim)
|
| 770 |
+
|
| 771 |
+
|
| 772 |
+
def _entmax_threshold_and_support(X, dim=-1):
|
| 773 |
+
Xsrt, _ = torch.sort(X, descending=True, dim=dim)
|
| 774 |
+
rho = _make_ix_like(X, dim)
|
| 775 |
+
mean = Xsrt.cumsum(dim) / rho
|
| 776 |
+
mean_sq = (Xsrt ** 2).cumsum(dim) / rho
|
| 777 |
+
ss = rho * (mean_sq - mean ** 2)
|
| 778 |
+
delta = (1 - ss) / rho
|
| 779 |
+
delta_nz = torch.clamp(delta, 0)
|
| 780 |
+
tau = mean - torch.sqrt(delta_nz)
|
| 781 |
+
support_size = (tau <= Xsrt).sum(dim).unsqueeze(dim)
|
| 782 |
+
# Guard: a degenerate/NaN row can give support_size == 0, so support_size - 1
|
| 783 |
+
# would be -1 and crash `gather` with a device-side index-out-of-bounds
|
| 784 |
+
# assert. Clamp to >=1 (falls back to the top element's threshold).
|
| 785 |
+
support_size = support_size.clamp(min=1)
|
| 786 |
+
tau_star = tau.gather(dim, support_size - 1)
|
| 787 |
+
return tau_star, support_size
|
| 788 |
+
|
| 789 |
+
|
| 790 |
+
class _Entmax15(torch.autograd.Function):
|
| 791 |
+
@staticmethod
|
| 792 |
+
def forward(ctx, X, dim):
|
| 793 |
+
ctx.dim = dim
|
| 794 |
+
max_val, _ = X.max(dim=dim, keepdim=True)
|
| 795 |
+
X = X - max_val # numerical stability, shift-invariant
|
| 796 |
+
X = X / 2 # exponent handling for alpha=1.5
|
| 797 |
+
tau_star, _ = _entmax_threshold_and_support(X, dim)
|
| 798 |
+
Y = torch.clamp(X - tau_star, min=0) ** 2
|
| 799 |
+
ctx.save_for_backward(Y)
|
| 800 |
+
return Y
|
| 801 |
+
|
| 802 |
+
@staticmethod
|
| 803 |
+
def backward(ctx, dY):
|
| 804 |
+
(Y,) = ctx.saved_tensors
|
| 805 |
+
gppr = Y.sqrt() # = clamp(X - tau, 0)
|
| 806 |
+
dX = dY * gppr
|
| 807 |
+
q = dX.sum(ctx.dim) / gppr.sum(ctx.dim).clamp_min(1e-12)
|
| 808 |
+
q = q.unsqueeze(ctx.dim)
|
| 809 |
+
dX = dX - q * gppr
|
| 810 |
+
return dX, None
|
| 811 |
+
|
| 812 |
+
|
| 813 |
+
def entmax15(X, dim=-1):
|
| 814 |
+
return _Entmax15.apply(X, dim)
|
| 815 |
+
|
| 816 |
+
|
| 817 |
+
# RoPE for the index projections (available at runtime in the assembled model
|
| 818 |
+
# dir; absent in a bare unit-test import -> rope is skipped).
|
| 819 |
+
try:
|
| 820 |
+
from ._sparse_base_impl import _llama_apply_rotary_pos_emb as _rope_fn
|
| 821 |
+
except Exception:
|
| 822 |
+
_rope_fn = None
|
| 823 |
+
|
| 824 |
+
# The router's norms are NOT in the checkpoint, so they are "missing keys" and get
|
| 825 |
+
# whatever `_init_weights` gives them. That hook only matches nn.Linear /
|
| 826 |
+
# nn.Embedding / _GemmaRMSNorm -- an unmatched module type is left as
|
| 827 |
+
# UNINITIALIZED memory, which is what produced `WEIGHT non-finite: nan=1` in the
|
| 828 |
+
# very first forward. Subclassing _GemmaRMSNorm makes the isinstance check hit,
|
| 829 |
+
# so the weight is zero-init'd (Gemma scale = 1 + weight => scale 1 at init).
|
| 830 |
+
try:
|
| 831 |
+
from ._sparse_base_impl import _GemmaRMSNorm as _RMSNormBase
|
| 832 |
+
except Exception:
|
| 833 |
+
class _RMSNormBase(nn.Module):
|
| 834 |
+
def __init__(self, dim, eps=1e-6):
|
| 835 |
+
super().__init__()
|
| 836 |
+
self.eps = eps
|
| 837 |
+
self.weight = nn.Parameter(torch.zeros(dim))
|
| 838 |
+
|
| 839 |
+
|
| 840 |
+
class _RMSNorm(_RMSNormBase):
|
| 841 |
+
"""Gemma-style RMSNorm (scale = 1 + weight, weight init 0) with fp32 +
|
| 842 |
+
finiteness guards for the router's index projections."""
|
| 843 |
+
|
| 844 |
+
def forward(self, x):
|
| 845 |
+
x32 = x.float()
|
| 846 |
+
# Sanitize BEFORE the variance reduction: a non-finite input makes
|
| 847 |
+
# mean(x^2)=inf -> rsqrt(inf)=0 -> inf*0=NaN, and that NaN is baked into
|
| 848 |
+
# the tensors saved for backward, so a post-hoc nan_to_num on the output
|
| 849 |
+
# cannot undo it (this is exactly the MulBackward0 NaN anomaly mode caught).
|
| 850 |
+
# Zeroing non-finite positions here keeps BOTH forward and backward finite.
|
| 851 |
+
x32 = torch.nan_to_num(x32, nan=0.0, posinf=0.0, neginf=0.0)
|
| 852 |
+
x32 = x32 * torch.rsqrt(x32.pow(2).mean(-1, keepdim=True) + self.eps)
|
| 853 |
+
x32 = x32.clamp(-_ACT_CLAMP, _ACT_CLAMP)
|
| 854 |
+
out = (x32 * (1.0 + self.weight.float())).clamp(
|
| 855 |
+
-_ACT_CLAMP, _ACT_CLAMP).to(x.dtype)
|
| 856 |
+
# The NaN anomaly fires INSIDE this mul's backward (grad wrt `weight`):
|
| 857 |
+
# incoming grads that are finite-but-huge overflow the sum over [B,S] to
|
| 858 |
+
# +-inf and cancel to NaN. Downstream hooks (idx_q/idx_k, post-RoPE)
|
| 859 |
+
# cannot prevent that, so bound the grad AT this output.
|
| 860 |
+
if out.requires_grad:
|
| 861 |
+
out.register_hook(_sanitize_grad)
|
| 862 |
+
if _DASH_DEBUG:
|
| 863 |
+
if not torch.isfinite(self.weight).all():
|
| 864 |
+
print(f"[DASH-DBG rmsnorm] WEIGHT non-finite: "
|
| 865 |
+
f"nan={int(torch.isnan(self.weight).sum())} "
|
| 866 |
+
f"inf={int(torch.isinf(self.weight).sum())}", flush=True)
|
| 867 |
+
_dbg_finite("rmsnorm_out", out)
|
| 868 |
+
_dbg_grad("rmsnorm_out(grad_in)", out)
|
| 869 |
+
return out
|
| 870 |
+
|
| 871 |
+
|
| 872 |
+
# ---------------------------------------------------------------------------
|
| 873 |
+
# DASH router / attention module
|
| 874 |
+
# ---------------------------------------------------------------------------
|
| 875 |
+
class DashAttention(nn.Module):
|
| 876 |
+
"""Differentiable adaptive sparse hierarchical attention.
|
| 877 |
+
|
| 878 |
+
Block scoring uses a SEPARATE low-dim index projection of ``hidden_states``
|
| 879 |
+
(the proven-stable Lightning-Indexer path that ``e2e.sh`` uses), NOT a
|
| 880 |
+
softmax summary over the raw attention keys. The raw-key summary averaged
|
| 881 |
+
over EVERY key position, so any non-finite key from the base model (dead /
|
| 882 |
+
special-token positions the sparse attention never reads -- which is why
|
| 883 |
+
sparse.sh is unaffected) turned a whole block's summary into NaN and poisoned
|
| 884 |
+
the router matmul's backward. The index projection avoids that entirely.
|
| 885 |
+
|
| 886 |
+
On top of the (non-local, causal) block-max scores we apply 1.5-entmax to get
|
| 887 |
+
a sparse, differentiable ``prior``; its top-k gives the block selection and
|
| 888 |
+
``log(prior)`` is fed as the differentiable per-block bias to the fused e2e
|
| 889 |
+
kernel -> router trains end-to-end from the LM loss, no KL distillation.
|
| 890 |
+
"""
|
| 891 |
+
|
| 892 |
+
def __init__(self, config):
|
| 893 |
+
super().__init__()
|
| 894 |
+
sp = getattr(config, "sparse_attention_config", {}) or {}
|
| 895 |
+
self.num_heads = config.num_attention_heads
|
| 896 |
+
self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
|
| 897 |
+
|
| 898 |
+
# block granularity reuses the base sparse knobs so configs stay aligned
|
| 899 |
+
self.block_size = int(sp.get("sparse_block_size", 128))
|
| 900 |
+
self.local_blocks = max(1, int(sp.get("sparse_local_block", 1)))
|
| 901 |
+
self.local_window = self.local_blocks * self.block_size
|
| 902 |
+
|
| 903 |
+
# DASH-specific knobs
|
| 904 |
+
self.scaling_factor = float(sp.get("dash_scaling_factor", 2.0)) # entmax sharpness
|
| 905 |
+
self.q_chunk = int(sp.get("dash_query_chunk", 2048)) # memory knob
|
| 906 |
+
self.image_only = bool(sp.get("dash_image_only", True))
|
| 907 |
+
self._image_token_id = int(sp.get("image_token_id", 128815))
|
| 908 |
+
# log-prior bias = log(prior)/sigma; prior_floor bounds the 1/p gradient.
|
| 909 |
+
self.sigma = float(sp.get("dash_sigma", 1.0))
|
| 910 |
+
self._prior_floor = float(sp.get("dash_prior_floor", 1e-2))
|
| 911 |
+
|
| 912 |
+
self.topk_blocks = int(sp.get("sparse_topk_blocks", 16))
|
| 913 |
+
self.remote_blocks = max(1, self.topk_blocks - self.local_blocks)
|
| 914 |
+
self.use_triton = bool(sp.get("dash_use_triton", True))
|
| 915 |
+
|
| 916 |
+
# single-head index projections (like DeepseekV2Indexer, minus KL).
|
| 917 |
+
self.index_dim = int(sp.get("sparse_index_dim", self.head_dim))
|
| 918 |
+
eps = getattr(config, "rms_norm_eps", 1e-6)
|
| 919 |
+
self.q_proj = nn.Linear(config.hidden_size, self.index_dim, bias=False)
|
| 920 |
+
self.k_proj = nn.Linear(config.hidden_size, self.index_dim, bias=False)
|
| 921 |
+
self.q_norm = _RMSNorm(self.index_dim, eps=eps)
|
| 922 |
+
self.k_norm = _RMSNorm(self.index_dim, eps=eps)
|
| 923 |
+
|
| 924 |
+
# Stage-0 summarization mode (env DASH_STAGE0):
|
| 925 |
+
# "indexer" (default) -> block-max of token-level idx_q.idx_k (current).
|
| 926 |
+
# "head_cls" -> paper Stage-0: a learned per-router summary
|
| 927 |
+
# query `head_cls` (zero-init => starts as mean
|
| 928 |
+
# pooling) attends the index-keys within each
|
| 929 |
+
# chunk to build one summary key k_bar_c, then
|
| 930 |
+
# the block score is idx_q . k_bar_c.
|
| 931 |
+
self.stage0 = os.environ.get("DASH_STAGE0", "indexer").lower()
|
| 932 |
+
# Learned summary query (zero-init: uniform local attention == mean pool
|
| 933 |
+
# at the start, then gradually learns an expressive chunk summary). Kept
|
| 934 |
+
# unconditionally so a checkpoint can switch modes without a shape change.
|
| 935 |
+
self.head_cls = nn.Parameter(torch.zeros(self.index_dim))
|
| 936 |
+
|
| 937 |
+
self._scale = 1.0 / math.sqrt(self.head_dim) # main attention
|
| 938 |
+
self._idx_scale = self.scaling_factor / math.sqrt(self.index_dim) # entmax input
|
| 939 |
+
self._neg = -1.0e4 # finite mask fill for entmax candidate masking
|
| 940 |
+
|
| 941 |
+
# Per-layer entmax temperature loop + the shared k schedule. Both are
|
| 942 |
+
# registered here so this layer gets its own scale/floor slot and its own
|
| 943 |
+
# ordinal (the L<sid> tag in every log line).
|
| 944 |
+
self._sched_id = _SCHED.register(self.scaling_factor, self._prior_floor)
|
| 945 |
+
_KS.register()
|
| 946 |
+
|
| 947 |
+
# Live handle on config.sparse_attention_config: the ROUTER STATE that
|
| 948 |
+
# inference must reproduce (entmax scale, prior floor, stage-0 mode) is
|
| 949 |
+
# partly runtime (the scheduler anneals scale/floor) and partly env
|
| 950 |
+
# (DASH_STAGE0). save_pretrained serializes this dict, so mirroring the
|
| 951 |
+
# current values into it is what makes the exported checkpoint
|
| 952 |
+
# self-describing -- otherwise inference re-reads the ORIGINAL
|
| 953 |
+
# dash_scaling_factor / dash_prior_floor and routes at a completely
|
| 954 |
+
# different temperature than the trained one.
|
| 955 |
+
# NOTE: the scale/floor are PER LAYER now, so the scalar keys carry the
|
| 956 |
+
# leader layer's values (backwards compatible) and the full profile goes
|
| 957 |
+
# to dash_scale_per_layer / dash_floor_per_layer. dash_topk_blocks is the
|
| 958 |
+
# k this checkpoint was trained at -- mid-anneal exports are NOT at K_END.
|
| 959 |
+
self._sp = sp if isinstance(sp, dict) else None
|
| 960 |
+
self._persist_router_cfg()
|
| 961 |
+
|
| 962 |
+
# -- export the effective router state into the saved config -------------
|
| 963 |
+
def _persist_router_cfg(self):
|
| 964 |
+
if self._sp is None or self._sched_id != _SCHED._leader:
|
| 965 |
+
return
|
| 966 |
+
self._sp["dash_stage0"] = self.stage0
|
| 967 |
+
self._sp["dash_router"] = "entmax_topk_perlayer"
|
| 968 |
+
self._sp["dash_sigma"] = float(self.sigma)
|
| 969 |
+
self._sp["dash_topk_blocks"] = int(_KS.k)
|
| 970 |
+
self._sp["dash_scaling_factor"] = float(
|
| 971 |
+
self._cur_idx_scale() * math.sqrt(self.index_dim))
|
| 972 |
+
self._sp["dash_prior_floor"] = float(self._cur_prior_floor())
|
| 973 |
+
self._sp["dash_scale_per_layer"] = [
|
| 974 |
+
float(_SCHED.scale(s, self.scaling_factor)) for s in sorted(_SCHED._scale)]
|
| 975 |
+
self._sp["dash_floor_per_layer"] = [
|
| 976 |
+
float(_SCHED.floor(s, self._prior_floor)) for s in sorted(_SCHED._floor)]
|
| 977 |
+
|
| 978 |
+
# -- scheduled knobs (identity when the scheduler is off) ---------------
|
| 979 |
+
def _cur_idx_scale(self):
|
| 980 |
+
return (_SCHED.scale(self._sched_id, self.scaling_factor)
|
| 981 |
+
/ math.sqrt(self.index_dim))
|
| 982 |
+
|
| 983 |
+
def _cur_prior_floor(self):
|
| 984 |
+
return _SCHED.floor(self._sched_id, self._prior_floor)
|
| 985 |
+
|
| 986 |
+
@torch.no_grad()
|
| 987 |
+
def build_image_block_mask(self, source_input_ids, key_length, nblk, device):
|
| 988 |
+
"""Return [B, nblk] bool: block contains >=1 image token. None if n/a."""
|
| 989 |
+
if source_input_ids is None or source_input_ids.ndim != 2:
|
| 990 |
+
return None
|
| 991 |
+
ids = source_input_ids[..., :key_length].to(device)
|
| 992 |
+
if ids.shape[-1] < key_length:
|
| 993 |
+
ids = F.pad(ids, (0, key_length - ids.shape[-1]), value=-1)
|
| 994 |
+
img = ids == self._image_token_id # [B, key_length]
|
| 995 |
+
if img.sum() == 0:
|
| 996 |
+
return None
|
| 997 |
+
Sp = nblk * self.block_size
|
| 998 |
+
if Sp > img.shape[-1]:
|
| 999 |
+
img = F.pad(img, (0, Sp - img.shape[-1]), value=False)
|
| 1000 |
+
else:
|
| 1001 |
+
img = img[..., :Sp]
|
| 1002 |
+
return img.view(img.shape[0], nblk, self.block_size).any(dim=-1) # [B,nblk]
|
| 1003 |
+
|
| 1004 |
+
# -- block scoring (indexer-style, non-local causal block-max) --------
|
| 1005 |
+
def _block_scores(self, hidden_states, position_embeddings, position_ids, nblk):
|
| 1006 |
+
"""[B,S,nblk] fp32 remote block scores; ``_SCORE_NEG`` where a block has no
|
| 1007 |
+
non-local causal key. Computed from a low-dim index projection (finite by
|
| 1008 |
+
construction), chunked over queries to bound memory."""
|
| 1009 |
+
B, S, _ = hidden_states.shape
|
| 1010 |
+
device = hidden_states.device
|
| 1011 |
+
# Router reads hidden_states directly; if an upstream layer emitted a
|
| 1012 |
+
# non-finite activation, the projections + matmul below manufacture NaN
|
| 1013 |
+
# block_scores (and NaN grads in q_norm/k_norm). Sanitize the router's own
|
| 1014 |
+
# view so it never propagates upstream NaN into the routing path.
|
| 1015 |
+
hs = torch.nan_to_num(hidden_states, nan=0.0, posinf=0.0, neginf=0.0)
|
| 1016 |
+
idx_q = self.q_norm(self.q_proj(hs).view(B, S, 1, self.index_dim)).transpose(1, 2)
|
| 1017 |
+
idx_k = self.k_norm(self.k_proj(hs).view(B, S, 1, self.index_dim)).transpose(1, 2)
|
| 1018 |
+
if _rope_fn is not None and position_embeddings is not None:
|
| 1019 |
+
cos, sin = position_embeddings
|
| 1020 |
+
idx_q, idx_k = _rope_fn(idx_q, idx_k, cos[..., :self.index_dim], sin[..., :self.index_dim])
|
| 1021 |
+
# nan_to_num defaults map +/-inf to +/-3.4e38, which then OVERFLOWS the
|
| 1022 |
+
# score matmul back into inf -> NaN. Map non-finite to 0 instead.
|
| 1023 |
+
idx_q = torch.nan_to_num(idx_q[:, 0].float(), nan=0.0, posinf=0.0, neginf=0.0) # [B,S,d]
|
| 1024 |
+
idx_k = torch.nan_to_num(idx_k[:, 0].float(), nan=0.0, posinf=0.0, neginf=0.0) # [B,S,d]
|
| 1025 |
+
# The amax/matmul backward below sits BETWEEN block_scores and the q/k_norm
|
| 1026 |
+
# RMSNorms and can manufacture NaN from the -inf entries masked_fill'd into
|
| 1027 |
+
# `sc` even when the incoming block_scores grad is finite -- which is why a
|
| 1028 |
+
# block_scores-level hook does not stop the k_norm MulBackward0 NaN. Report
|
| 1029 |
+
# (debug) then sanitize the grad right at the norm outputs (tight guard).
|
| 1030 |
+
_dbg_grad("idx_q(grad_in)", idx_q); _dbg_grad("idx_k(grad_in)", idx_k)
|
| 1031 |
+
if idx_q.requires_grad:
|
| 1032 |
+
idx_q.register_hook(_sanitize_grad)
|
| 1033 |
+
if idx_k.requires_grad:
|
| 1034 |
+
idx_k.register_hook(_sanitize_grad)
|
| 1035 |
+
kpos = torch.arange(S, device=device)
|
| 1036 |
+
Sp = nblk * self.block_size
|
| 1037 |
+
out = []
|
| 1038 |
+
for c0 in range(0, S, self.q_chunk):
|
| 1039 |
+
c1 = min(c0 + self.q_chunk, S)
|
| 1040 |
+
sc = torch.matmul(idx_q[:, c0:c1], idx_k.transpose(-1, -2)) # [B,Lq,S]
|
| 1041 |
+
sc = sc.clamp(-_ACT_CLAMP, _ACT_CLAMP)
|
| 1042 |
+
pos = position_ids[:, c0:c1] # [B,Lq]
|
| 1043 |
+
future = kpos[None, None, :] > pos[:, :, None]
|
| 1044 |
+
local = (kpos[None, None, :] > (pos[:, :, None] - self.local_window)) & ~future
|
| 1045 |
+
sc = sc.masked_fill(future | local, _SCORE_NEG) # remote-only
|
| 1046 |
+
if Sp > S:
|
| 1047 |
+
sc = F.pad(sc, (0, Sp - S), value=_SCORE_NEG)
|
| 1048 |
+
sc = sc.view(B, c1 - c0, nblk, self.block_size)
|
| 1049 |
+
out.append(sc.amax(dim=-1)) # [B,Lq,nblk]
|
| 1050 |
+
return torch.cat(out, dim=1) # [B,S,nblk]
|
| 1051 |
+
|
| 1052 |
+
# -- Stage-0 (head_cls): learned-summary block scoring (paper) --------
|
| 1053 |
+
def _block_scores_headcls(self, hidden_states, position_embeddings, position_ids, nblk):
|
| 1054 |
+
"""[B,S,nblk] fp32 remote block scores via a learned chunk summary.
|
| 1055 |
+
|
| 1056 |
+
Mirrors the paper's Stage-0/Stage-1: a per-router summary query
|
| 1057 |
+
``head_cls`` (zero-init -> starts as mean pooling) attends the index-keys
|
| 1058 |
+
within each chunk to form one summary key ``k_bar_c``; the block score is
|
| 1059 |
+
``idx_q_i . k_bar_c``. Same single-head index space (q_proj/k_proj/norms)
|
| 1060 |
+
and the same remote-only (causal minus local-window) masking as the
|
| 1061 |
+
``indexer`` path, so it drops straight into ``_route``. NOTE: the summary
|
| 1062 |
+
is over the INDEX-key space (not the main keys) to stay single-head and
|
| 1063 |
+
self-contained -- a deliberate adaptation of the paper to this codebase.
|
| 1064 |
+
"""
|
| 1065 |
+
B, S, _ = hidden_states.shape
|
| 1066 |
+
device = hidden_states.device
|
| 1067 |
+
hs = torch.nan_to_num(hidden_states, nan=0.0, posinf=0.0, neginf=0.0)
|
| 1068 |
+
idx_q = self.q_norm(self.q_proj(hs).view(B, S, 1, self.index_dim)).transpose(1, 2)
|
| 1069 |
+
idx_k = self.k_norm(self.k_proj(hs).view(B, S, 1, self.index_dim)).transpose(1, 2)
|
| 1070 |
+
if _rope_fn is not None and position_embeddings is not None:
|
| 1071 |
+
cos, sin = position_embeddings
|
| 1072 |
+
idx_q, idx_k = _rope_fn(idx_q, idx_k, cos[..., :self.index_dim], sin[..., :self.index_dim])
|
| 1073 |
+
idx_q = torch.nan_to_num(idx_q[:, 0].float(), nan=0.0, posinf=0.0, neginf=0.0) # [B,S,d]
|
| 1074 |
+
idx_k = torch.nan_to_num(idx_k[:, 0].float(), nan=0.0, posinf=0.0, neginf=0.0) # [B,S,d]
|
| 1075 |
+
if idx_q.requires_grad:
|
| 1076 |
+
idx_q.register_hook(_sanitize_grad)
|
| 1077 |
+
if idx_k.requires_grad:
|
| 1078 |
+
idx_k.register_hook(_sanitize_grad)
|
| 1079 |
+
d = self.index_dim
|
| 1080 |
+
inv_sqrt_d = 1.0 / math.sqrt(d)
|
| 1081 |
+
Sp = nblk * self.block_size
|
| 1082 |
+
|
| 1083 |
+
# Pad the index-keys up to whole chunks and track which slots are real.
|
| 1084 |
+
if Sp > S:
|
| 1085 |
+
idx_k = F.pad(idx_k, (0, 0, 0, Sp - S)) # [B,Sp,d]
|
| 1086 |
+
real = F.pad(idx_k.new_ones(B, S, dtype=torch.bool), (0, Sp - S), value=False)
|
| 1087 |
+
else:
|
| 1088 |
+
idx_k = idx_k[:, :Sp]
|
| 1089 |
+
real = idx_k.new_ones(B, Sp, dtype=torch.bool)
|
| 1090 |
+
kk = idx_k.view(B, nblk, self.block_size, d) # [B,nblk,bs,d]
|
| 1091 |
+
vmask = real.view(B, nblk, self.block_size) # [B,nblk,bs]
|
| 1092 |
+
|
| 1093 |
+
# Stage 0: head_cls local attention within each chunk -> summary key.
|
| 1094 |
+
# NaN-safe MASKED softmax: never feed -inf to softmax (a fully-padding
|
| 1095 |
+
# chunk would give all -inf -> NaN in BOTH forward AND SoftmaxBackward0,
|
| 1096 |
+
# which nan_to_num on the output cannot repair). Instead subtract the row
|
| 1097 |
+
# max (finite), exp, zero out padding multiplicatively, and normalize by a
|
| 1098 |
+
# clamped denominator -> forward and backward stay finite. hc=0 (zero-init)
|
| 1099 |
+
# -> uniform weights over valid tokens == mean pooling.
|
| 1100 |
+
hc = self.head_cls.to(kk.dtype).view(1, 1, 1, d)
|
| 1101 |
+
aw = (kk * hc).sum(-1) * inv_sqrt_d # [B,nblk,bs]
|
| 1102 |
+
aw = aw.clamp(-_ACT_CLAMP, _ACT_CLAMP)
|
| 1103 |
+
# NaN-safe masked softmax. Take the row-max over VALID tokens only: if we
|
| 1104 |
+
# max over padding too (padding aw==0), a chunk whose valid scores are all
|
| 1105 |
+
# negative would be dominated by padding, its valid exp() underflow, denom
|
| 1106 |
+
# -> 0, and the 1/denom in the division backward explodes (-> BmmBackward0
|
| 1107 |
+
# NaN downstream). Valid-only max keeps >=1 entry at e==1, so denom>=1.
|
| 1108 |
+
_neg = torch.finfo(aw.dtype).min
|
| 1109 |
+
row_max = torch.where(vmask, aw, aw.new_full((), _neg)).amax(dim=-1, keepdim=True)
|
| 1110 |
+
row_max = torch.where(vmask.any(dim=-1, keepdim=True), row_max,
|
| 1111 |
+
torch.zeros_like(row_max)) # all-pad chunk -> 0
|
| 1112 |
+
aw = aw - row_max
|
| 1113 |
+
e = torch.exp(aw) * vmask.to(aw.dtype) # padding -> 0
|
| 1114 |
+
w = e / e.sum(dim=-1, keepdim=True).clamp_min(1e-6) # [B,nblk,bs]
|
| 1115 |
+
k_bar = (w.unsqueeze(-1) * kk).sum(dim=2) # [B,nblk,d]
|
| 1116 |
+
# k_bar is a saved input of the Stage-1 matmul's backward -- force it
|
| 1117 |
+
# finite so BmmBackward0 can never emit NaN from a bad saved activation.
|
| 1118 |
+
k_bar = torch.nan_to_num(k_bar, nan=0.0, posinf=0.0, neginf=0.0)
|
| 1119 |
+
chunk_valid = vmask.any(dim=-1) # [B,nblk]
|
| 1120 |
+
|
| 1121 |
+
# Stage 1: z[i,c] = idx_q_i . k_bar_c, remote-only (causal minus local).
|
| 1122 |
+
blk_ids = torch.arange(nblk, device=device)
|
| 1123 |
+
out = []
|
| 1124 |
+
for c0 in range(0, S, self.q_chunk):
|
| 1125 |
+
c1 = min(c0 + self.q_chunk, S)
|
| 1126 |
+
z = torch.matmul(idx_q[:, c0:c1], k_bar.transpose(-1, -2)) # [B,Lq,nblk]
|
| 1127 |
+
# Sanitize the grad AT the matmul output: this is exactly the
|
| 1128 |
+
# grad_output BmmBackward0 consumes, so a NaN/Inf arriving from the
|
| 1129 |
+
# entmax/route backward (which the block_scores hook can miss under
|
| 1130 |
+
# gradient checkpointing) is cleaned before the matmul backward runs.
|
| 1131 |
+
if z.requires_grad:
|
| 1132 |
+
z.register_hook(_sanitize_grad)
|
| 1133 |
+
z = z * inv_sqrt_d
|
| 1134 |
+
z = z.clamp(-_ACT_CLAMP, _ACT_CLAMP)
|
| 1135 |
+
qblk = (position_ids[:, c0:c1] // self.block_size) # [B,Lq]
|
| 1136 |
+
# chunk c is a remote candidate iff it lies fully before the local
|
| 1137 |
+
# window: c <= qblk - local_blocks (matches the indexer path's
|
| 1138 |
+
# token-level `pos + local_blocks*block_size <= query_pos`).
|
| 1139 |
+
remote_ok = blk_ids[None, None, :] <= (qblk[:, :, None] - self.local_blocks)
|
| 1140 |
+
keep = remote_ok & chunk_valid[:, None, :]
|
| 1141 |
+
z = z.masked_fill(~keep, _SCORE_NEG)
|
| 1142 |
+
out.append(z)
|
| 1143 |
+
return torch.cat(out, dim=1) # [B,S,nblk]
|
| 1144 |
+
|
| 1145 |
+
def _route(self, block_scores, image_block_mask, block_cand=None):
|
| 1146 |
+
"""block_scores [B,S,nblk] -> (prior [B,S,nblk], cand [B,S,nblk] bool).
|
| 1147 |
+
|
| 1148 |
+
``block_cand`` [B,S,nblk] is the PER-QUERY candidate domain the caller builds
|
| 1149 |
+
from the indexer's page-aware ids (``_build_kl_candidate_mask`` -> same-page image
|
| 1150 |
+
bucket, with its own per-query fallback). It takes precedence over
|
| 1151 |
+
``image_block_mask`` [B,nblk], which is page-agnostic (image blocks of any page,
|
| 1152 |
+
identical for every query) and would drop page isolation entirely.
|
| 1153 |
+
"""
|
| 1154 |
+
# remote candidacy is now flagged by the finite sentinel, not by -inf
|
| 1155 |
+
remote = block_scores > (_SCORE_NEG * 0.5)
|
| 1156 |
+
cand = remote
|
| 1157 |
+
if block_cand is not None:
|
| 1158 |
+
cand = remote & block_cand
|
| 1159 |
+
elif self.image_only and image_block_mask is not None:
|
| 1160 |
+
cand = remote & image_block_mask[:, None, :]
|
| 1161 |
+
# per-query fallback: if image left no candidate but remote blocks exist,
|
| 1162 |
+
# drop the image constraint for that query.
|
| 1163 |
+
no_cand = ~cand.any(dim=-1, keepdim=True)
|
| 1164 |
+
cand = torch.where(no_cand, remote, cand)
|
| 1165 |
+
# one-shot per-layer calibration: land THIS layer's entmax support on the
|
| 1166 |
+
# current k budget, measured on real block scores (first training batch).
|
| 1167 |
+
if _SCHED.needs_calib(self._sched_id, self.training):
|
| 1168 |
+
_SCHED.calibrate(self._sched_id, block_scores.detach(), cand, self._neg,
|
| 1169 |
+
math.sqrt(self.index_dim))
|
| 1170 |
+
x = torch.where(cand, block_scores * self._cur_idx_scale(),
|
| 1171 |
+
block_scores.new_full((), self._neg))
|
| 1172 |
+
x = torch.nan_to_num(x, nan=self._neg, posinf=self._neg, neginf=self._neg)
|
| 1173 |
+
prior = entmax15(x, dim=-1) # [B,S,nblk]
|
| 1174 |
+
return prior, cand
|
| 1175 |
+
|
| 1176 |
+
def _select_and_bias(self, prior, cand, nblk):
|
| 1177 |
+
"""Hard top-k selection + the entmax log-prior bias on the kept blocks.
|
| 1178 |
+
|
| 1179 |
+
Returns (topi [B,S,K] int64, bias [B,S,K], valid [B,S,K] bool).
|
| 1180 |
+
|
| 1181 |
+
Selection is top-k over the MASKED PRIOR; entmax is monotone in its input
|
| 1182 |
+
within a row, so that is the same ranking as top-k over the block scores
|
| 1183 |
+
and the k highest-scoring candidates are always the ones read -- including
|
| 1184 |
+
blocks entmax pushed to exactly 0 when its support is narrower than k.
|
| 1185 |
+
Those carry bias log(prior_floor) - mu: attended, but no gradient, which is
|
| 1186 |
+
precisely what the per-layer temperature loop exists to prevent.
|
| 1187 |
+
|
| 1188 |
+
bias = (log(clamp(p, floor)) - mu)/sigma, mu = per-query MEAN over the kept
|
| 1189 |
+
log-priors (paper's log_g). Mean-centering is what lets an above-average
|
| 1190 |
+
remote block outweigh the zero-bias local window, instead of every remote
|
| 1191 |
+
block being uniformly down-weighted by log(p) <= 0.
|
| 1192 |
+
"""
|
| 1193 |
+
K = max(1, min(int(_KS.k), nblk))
|
| 1194 |
+
sel_prior = prior.masked_fill(~cand, -1.0) # non-candidates rank last
|
| 1195 |
+
topv, topi = sel_prior.topk(K, dim=-1) # [B,S,K]
|
| 1196 |
+
valid = topv >= 0.0 # a real candidate slot
|
| 1197 |
+
logp = torch.log(topv.clamp_min(self._cur_prior_floor()))
|
| 1198 |
+
cnt = valid.sum(dim=-1, keepdim=True).clamp(min=1)
|
| 1199 |
+
mu = torch.where(valid, logp, torch.zeros_like(logp)).sum(
|
| 1200 |
+
dim=-1, keepdim=True) / cnt
|
| 1201 |
+
bias = (logp - mu) * (1.0 / self.sigma)
|
| 1202 |
+
bias = torch.where(valid, bias, torch.zeros_like(bias)).clamp(
|
| 1203 |
+
-_ACT_CLAMP, _ACT_CLAMP)
|
| 1204 |
+
return topi, bias, valid
|
| 1205 |
+
|
| 1206 |
+
# -- Triton block-sparse forward (32k) --------------------------------
|
| 1207 |
+
def _forward_triton(self, q, k, v, topi, blk_bias, valid, S, nblk, position_ids):
|
| 1208 |
+
try:
|
| 1209 |
+
from .sparse_gather_e2e_triton import triton_gather_attention_e2e
|
| 1210 |
+
except Exception:
|
| 1211 |
+
return None
|
| 1212 |
+
# Fixed budget: the gathered width is exactly k for every query, every
|
| 1213 |
+
# layer, every step (padding slots -> index -1, skipped by the kernel).
|
| 1214 |
+
# A constant width also means the gradient-checkpoint recompute can never
|
| 1215 |
+
# disagree with the saved forward on metadata.
|
| 1216 |
+
block_indices = topi.masked_fill(~valid, -1).to(torch.int32).unsqueeze(1)
|
| 1217 |
+
out = triton_gather_attention_e2e(
|
| 1218 |
+
q, k, v, block_indices, blk_bias, position_ids, None,
|
| 1219 |
+
self.block_size, self._scale, self.local_window,
|
| 1220 |
+
)
|
| 1221 |
+
_dbg_grad("triton_out", out)
|
| 1222 |
+
if out is not None and not getattr(self, "_triton_logged", False):
|
| 1223 |
+
self._triton_logged = True
|
| 1224 |
+
print(f"[DASH] Triton block-sparse path active (S={S}, "
|
| 1225 |
+
f"block_size={self.block_size}, remote_topk={topi.shape[-1]}, "
|
| 1226 |
+
f"local_window={self.local_window})", flush=True)
|
| 1227 |
+
return out
|
| 1228 |
+
|
| 1229 |
+
# -- dense fallback (CPU / unsupported shape / validation reference) --
|
| 1230 |
+
def _forward_dense(self, q, k, v, topi, blk_bias, valid, S, nblk, attention_mask):
|
| 1231 |
+
B, H, _, D = q.shape
|
| 1232 |
+
device = q.device
|
| 1233 |
+
kidx = torch.arange(S, device=device)
|
| 1234 |
+
key_block = kidx // self.block_size
|
| 1235 |
+
# Scatter the per-selection bias back to a full [B,S,nblk] map so the dense
|
| 1236 |
+
# reference attends EXACTLY the same k blocks as the kernel; unselected
|
| 1237 |
+
# entries carry _MASK_NEG and are dropped below. `scatter` (functional)
|
| 1238 |
+
# keeps the autograd path into blk_bias.
|
| 1239 |
+
src = torch.where(valid, blk_bias, blk_bias.new_full((), _MASK_NEG))
|
| 1240 |
+
logp = torch.full((B, blk_bias.shape[1], nblk), _MASK_NEG,
|
| 1241 |
+
device=device, dtype=blk_bias.dtype).scatter(-1, topi, src)
|
| 1242 |
+
out_chunks = []
|
| 1243 |
+
for qs in range(0, S, self.q_chunk):
|
| 1244 |
+
qe = min(qs + self.q_chunk, S)
|
| 1245 |
+
qidx = torch.arange(qs, qe, device=device)
|
| 1246 |
+
q_c = q[:, :, qs:qe, :]
|
| 1247 |
+
base = logp[:, qs:qe].index_select(-1, key_block)[:, None] # [B,1,Lq,S]
|
| 1248 |
+
causal = (kidx[None, :] <= qidx[:, None])[None, None]
|
| 1249 |
+
local = ((kidx[None, :] <= qidx[:, None]) &
|
| 1250 |
+
(kidx[None, :] > (qidx[:, None] - self.local_window)))[None, None]
|
| 1251 |
+
sel = base > (_MASK_NEG * 0.5)
|
| 1252 |
+
keep = causal & (local | sel)
|
| 1253 |
+
mask = torch.where(local, torch.zeros_like(base), base)
|
| 1254 |
+
mask = mask.masked_fill(~keep, _MASK_NEG)
|
| 1255 |
+
if attention_mask is not None:
|
| 1256 |
+
mask = mask + attention_mask[:, :, qs:qe, :S].to(mask.dtype)
|
| 1257 |
+
mask = mask.clamp_min(_MASK_NEG).to(q.dtype)
|
| 1258 |
+
out_chunks.append(torch.nn.functional.scaled_dot_product_attention(
|
| 1259 |
+
q_c, k, v, attn_mask=mask, is_causal=False, scale=self._scale))
|
| 1260 |
+
return torch.cat(out_chunks, dim=2)
|
| 1261 |
+
|
| 1262 |
+
# -- forward -----------------------------------------------------------
|
| 1263 |
+
def forward(self, hidden_states, q, k, v, position_embeddings=None,
|
| 1264 |
+
position_ids=None, attention_mask=None, image_block_mask=None,
|
| 1265 |
+
block_cand=None, page_ids=None):
|
| 1266 |
+
"""hidden_states [B,S,hidden]; q,k,v [B,H,S,D] (RoPE'd, GQA-expanded).
|
| 1267 |
+
|
| 1268 |
+
Returns [B,H,S,D] (before o_proj). image_block_mask: [B,nblk] bool or None.
|
| 1269 |
+
block_cand: [B,S,nblk] bool per-query candidate domain (page isolation) or None.
|
| 1270 |
+
page_ids: (image_token_page_ids, query_page_ids) -- used only by the
|
| 1271 |
+
``DASH_LOG_PAGE_EVERY`` diagnostics, never in the computation.
|
| 1272 |
+
"""
|
| 1273 |
+
B, H, S, D = q.shape
|
| 1274 |
+
nblk = (S + self.block_size - 1) // self.block_size
|
| 1275 |
+
device = q.device
|
| 1276 |
+
if position_ids is None:
|
| 1277 |
+
position_ids = torch.arange(S, device=device)[None].expand(B, -1)
|
| 1278 |
+
|
| 1279 |
+
# Refresh THIS layer's entmax temperature and the shared k (once per
|
| 1280 |
+
# micro-batch each; both stay frozen for the rest of the forward).
|
| 1281 |
+
_KS.advance(self._sched_id, self.training)
|
| 1282 |
+
_SCHED.advance(self._sched_id, self.training)
|
| 1283 |
+
# Mirror the current scale/floor/k into the config so any checkpoint saved
|
| 1284 |
+
# after this step carries the router state inference has to reproduce.
|
| 1285 |
+
self._persist_router_cfg()
|
| 1286 |
+
|
| 1287 |
+
if self.stage0 == "head_cls":
|
| 1288 |
+
block_scores = self._block_scores_headcls(hidden_states, position_embeddings, position_ids, nblk)
|
| 1289 |
+
else:
|
| 1290 |
+
block_scores = self._block_scores(hidden_states, position_embeddings, position_ids, nblk)
|
| 1291 |
+
if not getattr(self, "_stage0_logged", False):
|
| 1292 |
+
self._stage0_logged = True
|
| 1293 |
+
print(f"[DASH] Stage-0 summarization = {self.stage0!r}", flush=True)
|
| 1294 |
+
# Router gradient choke point. `prior`'s own hook (below) only cleans the
|
| 1295 |
+
# grad ENTERING prior; the entmax / where / log backward BETWEEN prior and
|
| 1296 |
+
# block_scores can still manufacture NaN/Inf on degenerate rows, which then
|
| 1297 |
+
# flows into q_norm/k_norm (the MulBackward0 anomaly that persists even
|
| 1298 |
+
# after the forward was made finite). Sanitizing here -- upstream of the
|
| 1299 |
+
# RMSNorm backward -- guarantees only finite grads reach q_proj/k_proj.
|
| 1300 |
+
if block_scores.requires_grad:
|
| 1301 |
+
block_scores.register_hook(_sanitize_grad)
|
| 1302 |
+
prior, cand = self._route(block_scores, image_block_mask, block_cand) # [B,S,nblk]
|
| 1303 |
+
topi, blk_bias, valid = self._select_and_bias(prior, cand, nblk)
|
| 1304 |
+
_dbg_finite("block_scores", block_scores); _dbg_finite("prior", prior)
|
| 1305 |
+
_dbg_grad("prior", prior)
|
| 1306 |
+
# `prior` is the SINGLE gradient channel from the LM loss back to the router
|
| 1307 |
+
# (loss -> kernel d_bias / dense mask -> blk_bias/log(prior) -> prior ->
|
| 1308 |
+
# block_scores -> q_proj/k_proj). The kernel's d_bias backward on our
|
| 1309 |
+
# log(prior) bias can emit NaN/Inf for degenerate selections (finite loss,
|
| 1310 |
+
# NaN grad_norm). Sanitizing the gradient AT this choke point keeps the
|
| 1311 |
+
# router params finite while still training them from finite gradients.
|
| 1312 |
+
if prior.requires_grad:
|
| 1313 |
+
prior.register_hook(_sanitize_grad)
|
| 1314 |
+
|
| 1315 |
+
if _LOG_SPARSITY:
|
| 1316 |
+
_log_train_sparsity(prior, cand, valid, self.block_size, self._sched_id)
|
| 1317 |
+
if (_LOG_PAGE_EVERY and page_ids is not None and block_cand is not None
|
| 1318 |
+
and self.training and torch._C._current_graph_task_id() == -1):
|
| 1319 |
+
self._page_log_n = getattr(self, "_page_log_n", 0) + 1
|
| 1320 |
+
if self._page_log_n % _LOG_PAGE_EVERY == 0 or self._page_log_n == 1:
|
| 1321 |
+
_log_page_purity(topi, valid, block_cand,
|
| 1322 |
+
block_scores.detach() > (_SCORE_NEG * 0.5),
|
| 1323 |
+
page_ids[0], page_ids[1], self.block_size, nblk,
|
| 1324 |
+
self._sched_id, self._page_log_n)
|
| 1325 |
+
# How much of the paid-for k blocks the attention really uses. Skipped in
|
| 1326 |
+
# the gradient-checkpoint recompute (same guard as the scheduler), else a
|
| 1327 |
+
# layer's counter advances twice per micro-batch.
|
| 1328 |
+
if _LOG_NEFF_EVERY and self.training and \
|
| 1329 |
+
torch._C._current_graph_task_id() == -1:
|
| 1330 |
+
_log_neff(q, k, topi, blk_bias, valid, position_ids, self.block_size,
|
| 1331 |
+
self.local_window, self._scale, self._sched_id)
|
| 1332 |
+
# Oracle recall of the routed set (block/score recall vs. the dense
|
| 1333 |
+
# attention mass). Same recompute guard: the [H,rows,S] qk is expensive
|
| 1334 |
+
# enough that running it twice per micro-batch is not free.
|
| 1335 |
+
if _LOG_RECALL_EVERY and self.training and \
|
| 1336 |
+
torch._C._current_graph_task_id() == -1:
|
| 1337 |
+
_log_oracle_recall(q, k, topi, valid, cand, position_ids,
|
| 1338 |
+
self.block_size, self._scale, self._sched_id)
|
| 1339 |
+
# this layer's own closed loop reads back its entmax support size
|
| 1340 |
+
_SCHED.observe(self._sched_id, prior, cand, self.training)
|
| 1341 |
+
|
| 1342 |
+
if self.use_triton and q.is_cuda and attention_mask is None:
|
| 1343 |
+
out = self._forward_triton(q, k, v, topi, blk_bias, valid, S, nblk,
|
| 1344 |
+
position_ids)
|
| 1345 |
+
if out is not None:
|
| 1346 |
+
return out
|
| 1347 |
+
|
| 1348 |
+
return self._forward_dense(q, k, v, topi, blk_bias, valid, S, nblk,
|
| 1349 |
+
attention_mask)
|
deepencoder.py
ADDED
|
@@ -0,0 +1,1072 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch.nn as nn
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
import copy
|
| 5 |
+
|
| 6 |
+
from contextlib import nullcontext
|
| 7 |
+
import math
|
| 8 |
+
from typing import Optional, Tuple
|
| 9 |
+
# from megatron.model import LayerNorm
|
| 10 |
+
|
| 11 |
+
from einops import rearrange
|
| 12 |
+
from easydict import EasyDict as adict
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
from typing import Optional, Tuple, Type
|
| 16 |
+
from functools import partial
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class MlpProjector(nn.Module):
|
| 21 |
+
|
| 22 |
+
def __init__(self, cfg):
|
| 23 |
+
|
| 24 |
+
super().__init__()
|
| 25 |
+
|
| 26 |
+
self.cfg = cfg
|
| 27 |
+
|
| 28 |
+
if cfg.projector_type == "identity":
|
| 29 |
+
modules = nn.Identity()
|
| 30 |
+
|
| 31 |
+
elif cfg.projector_type == "linear":
|
| 32 |
+
modules = nn.Linear(cfg.input_dim, cfg.n_embed)
|
| 33 |
+
|
| 34 |
+
elif cfg.projector_type == "mlp_gelu":
|
| 35 |
+
mlp_depth = cfg.get("depth", 1)
|
| 36 |
+
modules = [nn.Linear(cfg.input_dim, cfg.n_embed)]
|
| 37 |
+
for _ in range(1, mlp_depth):
|
| 38 |
+
modules.append(nn.GELU())
|
| 39 |
+
modules.append(nn.Linear(cfg.n_embed, cfg.n_embed))
|
| 40 |
+
modules = nn.Sequential(*modules)
|
| 41 |
+
|
| 42 |
+
elif cfg.projector_type == "normlayer_downsample_mlp_gelu":
|
| 43 |
+
mlp_depth = cfg.get("depth", 1)
|
| 44 |
+
mlp_ratio = cfg.get("mlp_ratio", 1)
|
| 45 |
+
modules = [
|
| 46 |
+
nn.LayerNorm(cfg.input_dim * cfg.downsample_ratio * cfg.downsample_ratio),
|
| 47 |
+
nn.Linear(cfg.input_dim * cfg.downsample_ratio * cfg.downsample_ratio, cfg.n_embed * mlp_ratio)
|
| 48 |
+
]
|
| 49 |
+
for _ in range(1, mlp_depth - 1):
|
| 50 |
+
modules.append(nn.GELU())
|
| 51 |
+
modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed * mlp_ratio))
|
| 52 |
+
modules.append(nn.GELU())
|
| 53 |
+
modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed))
|
| 54 |
+
modules = nn.Sequential(*modules)
|
| 55 |
+
|
| 56 |
+
elif cfg.projector_type == "downsample_mlp_gelu":
|
| 57 |
+
mlp_depth = cfg.get("depth", 1)
|
| 58 |
+
mlp_ratio = cfg.get("mlp_ratio", 1)
|
| 59 |
+
modules = [nn.Linear(cfg.input_dim * cfg.downsample_ratio * cfg.downsample_ratio, cfg.n_embed * mlp_ratio)]
|
| 60 |
+
for _ in range(1, mlp_depth - 1):
|
| 61 |
+
modules.append(nn.GELU())
|
| 62 |
+
modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed * mlp_ratio))
|
| 63 |
+
modules.append(nn.GELU())
|
| 64 |
+
modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed))
|
| 65 |
+
modules = nn.Sequential(*modules)
|
| 66 |
+
|
| 67 |
+
elif cfg.projector_type == "low_high_hybrid_split_mlp_gelu":
|
| 68 |
+
mlp_depth = cfg.get("depth", 1)
|
| 69 |
+
self.high_up_proj = nn.Linear(cfg.input_dim, cfg.n_embed // 2)
|
| 70 |
+
self.low_up_proj = nn.Linear(cfg.input_dim, cfg.n_embed // 2)
|
| 71 |
+
|
| 72 |
+
modules = []
|
| 73 |
+
for _ in range(1, mlp_depth):
|
| 74 |
+
modules.append(nn.GELU())
|
| 75 |
+
modules.append(nn.Linear(cfg.n_embed, cfg.n_embed))
|
| 76 |
+
modules = nn.Sequential(*modules)
|
| 77 |
+
|
| 78 |
+
elif cfg.projector_type == "hybrid_split_feature_mlp_gelu":
|
| 79 |
+
mlp_depth = cfg.get("depth", 1)
|
| 80 |
+
channel_div = cfg.get("channel_div", 0.5)
|
| 81 |
+
self.high_up_proj = nn.Linear(cfg.input_dim[0], int(cfg.n_embed * channel_div))
|
| 82 |
+
self.low_up_proj = nn.Linear(cfg.input_dim[1], cfg.n_embed - int(cfg.n_embed * channel_div))
|
| 83 |
+
|
| 84 |
+
modules = []
|
| 85 |
+
for _ in range(1, mlp_depth):
|
| 86 |
+
modules.append(nn.GELU())
|
| 87 |
+
modules.append(nn.Linear(cfg.n_embed, cfg.n_embed))
|
| 88 |
+
modules = nn.Sequential(*modules)
|
| 89 |
+
|
| 90 |
+
elif cfg.projector_type == "low_high_split_mlp_gelu":
|
| 91 |
+
mlp_depth = cfg.get("depth", 1)
|
| 92 |
+
modules = []
|
| 93 |
+
for _ in range(1, mlp_depth):
|
| 94 |
+
modules.append(nn.GELU())
|
| 95 |
+
modules.append(nn.Linear(cfg.n_embed // 2, cfg.n_embed // 2))
|
| 96 |
+
modules = nn.Sequential(*modules)
|
| 97 |
+
self.high_layers = nn.Sequential(*modules)
|
| 98 |
+
self.low_layers = copy.deepcopy(modules)
|
| 99 |
+
|
| 100 |
+
else:
|
| 101 |
+
raise ValueError(f"Unknown projector type: {cfg.projector_type}")
|
| 102 |
+
|
| 103 |
+
if cfg.get("token_pooling", False):
|
| 104 |
+
self.token_pooling_layer = nn.Linear(cfg.input_dim * 4, cfg.input_dim)
|
| 105 |
+
|
| 106 |
+
if cfg.get("conv_fusion_high_low_features", False):
|
| 107 |
+
self.fusion_layer = nn.Linear(cfg.input_dim, cfg.input_dim)
|
| 108 |
+
self.layers = modules
|
| 109 |
+
|
| 110 |
+
def forward(self, x):
|
| 111 |
+
if self.cfg.get("token_pooling", False):
|
| 112 |
+
batch_size, wxh, channels = x.shape
|
| 113 |
+
w = h = int(wxh**0.5)
|
| 114 |
+
x = x.view(batch_size, w, h, channels)
|
| 115 |
+
x = x.permute(0, 3, 1, 2)
|
| 116 |
+
# import ipdb; ipdb.set_trace()
|
| 117 |
+
patches = x.unfold(2, 2, 2).unfold(3, 2, 2)
|
| 118 |
+
batch_size, channels, h_patches, w_patches, _, _ = patches.size()
|
| 119 |
+
patches = patches.contiguous().view(batch_size, channels, h_patches * w_patches, -1)
|
| 120 |
+
|
| 121 |
+
patches = patches.permute(0, 2, 1, 3).contiguous()
|
| 122 |
+
patches = patches.view(batch_size, h_patches * w_patches, channels * 4)
|
| 123 |
+
|
| 124 |
+
x = self.token_pooling_layer(patches)
|
| 125 |
+
|
| 126 |
+
if self.cfg.get("conv_fusion_high_low_features", False):
|
| 127 |
+
x = self.fusion_layer(x[:, 0]) + x[:, 1]
|
| 128 |
+
|
| 129 |
+
if self.cfg.projector_type == 'low_high_hybrid_split_mlp_gelu':
|
| 130 |
+
high_x, low_x = x[0], x[1]
|
| 131 |
+
high_x = self.high_up_proj(high_x)
|
| 132 |
+
low_x = self.low_up_proj(low_x)
|
| 133 |
+
x = torch.concat([high_x, low_x], dim=-1)
|
| 134 |
+
|
| 135 |
+
if self.cfg.projector_type == 'hybrid_split_feature_mlp_gelu':
|
| 136 |
+
high_x = x[...,:self.cfg.input_dim[0]]
|
| 137 |
+
low_x = x[...,self.cfg.input_dim[0]:]
|
| 138 |
+
high_x = self.high_up_proj(high_x)
|
| 139 |
+
low_x = self.low_up_proj(low_x)
|
| 140 |
+
x = torch.concat([high_x, low_x], dim=-1)
|
| 141 |
+
|
| 142 |
+
if self.cfg.projector_type == 'low_high_split_mlp_gelu':
|
| 143 |
+
high_x, low_x = x[0], x[1]
|
| 144 |
+
high_x = self.high_layers(high_x)
|
| 145 |
+
low_x = self.low_layers(low_x)
|
| 146 |
+
x = torch.concat([high_x, low_x], dim=-1)
|
| 147 |
+
return x
|
| 148 |
+
|
| 149 |
+
if self.cfg.projector_type == 'downsample_mlp_gelu' or self.cfg.projector_type == 'normlayer_downsample_mlp_gelu':
|
| 150 |
+
bs, hw, input_dim = x.shape
|
| 151 |
+
h = w = int((hw) ** 0.5)
|
| 152 |
+
|
| 153 |
+
"""compute padding"""
|
| 154 |
+
if h % self.cfg.downsample_ratio:
|
| 155 |
+
pad = self.cfg.downsample_ratio - h % self.cfg.downsample_ratio
|
| 156 |
+
else:
|
| 157 |
+
pad = 0
|
| 158 |
+
x = x.reshape(bs, h, w, input_dim)
|
| 159 |
+
if pad > 0:
|
| 160 |
+
x = F.pad(x, (0, 0, 0, pad, 0, pad), "constant", 0)
|
| 161 |
+
|
| 162 |
+
"""4 to 1 concat"""
|
| 163 |
+
x = x.permute(0, 3, 1, 2) # B, C, H, W
|
| 164 |
+
x = F.unfold(x, kernel_size=self.cfg.downsample_ratio, stride=self.cfg.downsample_ratio, padding=0) # B, C*4, HW // 4
|
| 165 |
+
x = x.permute(0, 2, 1)
|
| 166 |
+
|
| 167 |
+
return self.layers(x)
|
| 168 |
+
|
| 169 |
+
@staticmethod
|
| 170 |
+
def get_flops_per_sample(cfg):
|
| 171 |
+
if cfg.projector_type == "linear":
|
| 172 |
+
fwd = 2 * cfg.input_dim * cfg.n_embed
|
| 173 |
+
|
| 174 |
+
elif "mlp_gelu" in cfg.projector_type :
|
| 175 |
+
mlp_depth = cfg.get("depth", 1)
|
| 176 |
+
downsample_ratio = cfg.get("downsample_ratio", 1)
|
| 177 |
+
input_dim = sum(cfg.input_dim) if isinstance(cfg.input_dim, list) else cfg.input_dim
|
| 178 |
+
input_dim = input_dim * downsample_ratio * downsample_ratio
|
| 179 |
+
fwd = 2 * input_dim * cfg.n_embed + (mlp_depth - 1) * 2 * cfg.n_embed * cfg.n_embed
|
| 180 |
+
else:
|
| 181 |
+
fwd = 0
|
| 182 |
+
|
| 183 |
+
return fwd * 3
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
#===================clip============================================================
|
| 187 |
+
|
| 188 |
+
class LayerNormfp32(torch.nn.LayerNorm):
|
| 189 |
+
"""Subclass torch's LayerNorm to handle fp16."""
|
| 190 |
+
|
| 191 |
+
def forward(self, x: torch.Tensor):
|
| 192 |
+
orig_type = x.dtype
|
| 193 |
+
ret = super().forward(x.type(torch.float32))
|
| 194 |
+
return ret.type(orig_type)
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def get_abs_pos(abs_pos, tgt_size):
|
| 198 |
+
# abs_pos: L, C
|
| 199 |
+
# tgt_size: M
|
| 200 |
+
# return: M, C
|
| 201 |
+
|
| 202 |
+
# print(tgt_size)
|
| 203 |
+
# print(abs_pos.shape)
|
| 204 |
+
# exit()
|
| 205 |
+
dim = abs_pos.size(-1)
|
| 206 |
+
# print(dim)
|
| 207 |
+
abs_pos_new = abs_pos.squeeze(0)
|
| 208 |
+
cls_token, old_pos_embed = abs_pos_new[:1], abs_pos_new[1:]
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
src_size = int(math.sqrt(abs_pos_new.shape[0] - 1))
|
| 213 |
+
tgt_size = int(math.sqrt(tgt_size))
|
| 214 |
+
dtype = abs_pos.dtype
|
| 215 |
+
|
| 216 |
+
if src_size != tgt_size:
|
| 217 |
+
old_pos_embed = old_pos_embed.view(1, src_size, src_size, dim).permute(0, 3, 1,
|
| 218 |
+
2).contiguous()
|
| 219 |
+
old_pos_embed = old_pos_embed.to(torch.float32)
|
| 220 |
+
new_pos_embed = F.interpolate(
|
| 221 |
+
old_pos_embed,
|
| 222 |
+
size=(tgt_size, tgt_size),
|
| 223 |
+
mode='bicubic',
|
| 224 |
+
antialias=True,
|
| 225 |
+
align_corners=False,
|
| 226 |
+
).to(dtype)
|
| 227 |
+
new_pos_embed = new_pos_embed.permute(0, 2, 3, 1)
|
| 228 |
+
new_pos_embed = new_pos_embed.view(tgt_size * tgt_size, dim)
|
| 229 |
+
vision_pos_embed = torch.cat([cls_token, new_pos_embed], dim=0)
|
| 230 |
+
vision_pos_embed = vision_pos_embed.view(1, tgt_size * tgt_size + 1, dim)
|
| 231 |
+
return vision_pos_embed
|
| 232 |
+
else:
|
| 233 |
+
return abs_pos
|
| 234 |
+
|
| 235 |
+
@torch.jit.script
|
| 236 |
+
def quick_gelu(x):
|
| 237 |
+
return x * torch.sigmoid(1.702 * x)
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
class CLIPVisionEmbeddings(nn.Module):
|
| 242 |
+
def __init__(self, hidden_size=1024, image_size=224, patch_size=14, num_channels=3):
|
| 243 |
+
super().__init__()
|
| 244 |
+
self.embed_dim = hidden_size
|
| 245 |
+
self.image_size = image_size
|
| 246 |
+
self.patch_size = patch_size
|
| 247 |
+
|
| 248 |
+
self.class_embedding = torch.nn.Parameter(torch.randn(self.embed_dim))
|
| 249 |
+
|
| 250 |
+
self.patch_embedding = torch.nn.Conv2d(
|
| 251 |
+
in_channels=num_channels,
|
| 252 |
+
out_channels=self.embed_dim,
|
| 253 |
+
kernel_size=self.patch_size,
|
| 254 |
+
stride=self.patch_size,
|
| 255 |
+
bias=False,
|
| 256 |
+
)
|
| 257 |
+
|
| 258 |
+
self.num_patches = (self.image_size // self.patch_size) ** 2
|
| 259 |
+
self.num_positions = self.num_patches + 1
|
| 260 |
+
self.position_embedding = torch.nn.Embedding(self.num_positions, self.embed_dim)
|
| 261 |
+
self.register_buffer(
|
| 262 |
+
"position_ids", torch.arange(self.num_positions).expand((1, -1))
|
| 263 |
+
)
|
| 264 |
+
|
| 265 |
+
def forward(self, pixel_values, patch_embeds):
|
| 266 |
+
batch_size = pixel_values.shape[0]
|
| 267 |
+
# patch_embeds = self.patch_embedding(
|
| 268 |
+
# pixel_values
|
| 269 |
+
# ) # shape = [*, width, grid, grid]
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
if patch_embeds is not None:
|
| 273 |
+
patch_embeds = patch_embeds
|
| 274 |
+
# print(patch_embeds.shape)
|
| 275 |
+
else:
|
| 276 |
+
patch_embeds = self.patch_embedding(pixel_values)
|
| 277 |
+
# print(111111)
|
| 278 |
+
# shape = [*, width, grid, grid]
|
| 279 |
+
# patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
|
| 280 |
+
|
| 281 |
+
patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
class_embeds = self.class_embedding.expand(batch_size, 1, -1)
|
| 285 |
+
embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
|
| 286 |
+
|
| 287 |
+
# x = torch.cat([cls_token, x], dim=1)
|
| 288 |
+
embeddings = embeddings + get_abs_pos(self.position_embedding(self.position_ids), embeddings.size(1))
|
| 289 |
+
# embeddings = embeddings + self.position_embedding(self.position_ids)
|
| 290 |
+
return embeddings
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
class NoTPFeedForward(nn.Module):
|
| 294 |
+
def __init__(
|
| 295 |
+
self,
|
| 296 |
+
cfg,
|
| 297 |
+
dim: int,
|
| 298 |
+
hidden_dim: int,
|
| 299 |
+
):
|
| 300 |
+
super().__init__()
|
| 301 |
+
|
| 302 |
+
self.fc1 = torch.nn.Linear(dim, hidden_dim, bias=True)
|
| 303 |
+
self.fc2 = torch.nn.Linear(hidden_dim, dim, bias=True)
|
| 304 |
+
|
| 305 |
+
def forward(self, x):
|
| 306 |
+
output = self.fc2(quick_gelu(self.fc1(x)))
|
| 307 |
+
return output
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
class NoTPAttention(torch.nn.Module):
|
| 313 |
+
def __init__(self, cfg):
|
| 314 |
+
super().__init__()
|
| 315 |
+
self.num_heads = cfg.num_attention_heads
|
| 316 |
+
self.n_local_heads = cfg.num_attention_heads
|
| 317 |
+
self.head_dim = cfg.hidden_size // cfg.num_attention_heads
|
| 318 |
+
self.max_seq_len = cfg.seq_length
|
| 319 |
+
self.use_flash_attention = cfg.use_flash_attn
|
| 320 |
+
|
| 321 |
+
self.qkv_proj = torch.nn.Linear(cfg.hidden_size, cfg.hidden_size * 3, bias=True)
|
| 322 |
+
self.out_proj = torch.nn.Linear(cfg.hidden_size, cfg.hidden_size, bias=True)
|
| 323 |
+
|
| 324 |
+
# self.core_attention = CoreAttention(cfg, AttnType.self_attn)
|
| 325 |
+
|
| 326 |
+
self.attn_drop = cfg.attention_dropout
|
| 327 |
+
|
| 328 |
+
def forward(
|
| 329 |
+
self,
|
| 330 |
+
x: torch.Tensor,
|
| 331 |
+
):
|
| 332 |
+
bsz, seqlen, _ = x.shape
|
| 333 |
+
xqkv = self.qkv_proj(x)
|
| 334 |
+
xqkv = xqkv.view(bsz, seqlen, 3, self.num_heads, self.head_dim)
|
| 335 |
+
|
| 336 |
+
if self.use_flash_attention:
|
| 337 |
+
|
| 338 |
+
xq, xk, xv = torch.split(xqkv, 1, dim=2)
|
| 339 |
+
xq = xq.squeeze(2)
|
| 340 |
+
xk = xk.squeeze(2)
|
| 341 |
+
xv = xv.squeeze(2)
|
| 342 |
+
# xq, xk, xv = xqkv[:, :, 0, ...], xqkv[:, :, 1, ...], xqkv[:, :, 2, ...]
|
| 343 |
+
|
| 344 |
+
# (B, num_head, S, head_size)
|
| 345 |
+
xq = xq.permute(0, 2, 1, 3)
|
| 346 |
+
xk = xk.permute(0, 2, 1, 3)
|
| 347 |
+
xv = xv.permute(0, 2, 1, 3)
|
| 348 |
+
# with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):
|
| 349 |
+
output = torch.nn.functional.scaled_dot_product_attention(xq, xk, xv, attn_mask=None)
|
| 350 |
+
output = output.permute(0, 2, 1, 3).reshape(bsz, seqlen, -1)
|
| 351 |
+
# output = output.permute(0, 2, 1, 3).contiguous().view(bsz, seqlen, -1)
|
| 352 |
+
else:
|
| 353 |
+
# print(22222)
|
| 354 |
+
xq, xk, xv = torch.split(xqkv, 1, dim=2)
|
| 355 |
+
xq = xq.squeeze(2)
|
| 356 |
+
xk = xk.squeeze(2)
|
| 357 |
+
xv = xv.squeeze(2)
|
| 358 |
+
# xq, xk, xv = xqkv[:, :, 0, ...], xqkv[:, :, 1, ...], xqkv[:, :, 2, ...]
|
| 359 |
+
|
| 360 |
+
# (B, num_head, S, head_size)
|
| 361 |
+
xq = xq.permute(0, 2, 1, 3)
|
| 362 |
+
xk = xk.permute(0, 2, 1, 3)
|
| 363 |
+
xv = xv.permute(0, 2, 1, 3)
|
| 364 |
+
# with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):
|
| 365 |
+
output = torch.nn.functional.scaled_dot_product_attention(xq, xk, xv, attn_mask=None)
|
| 366 |
+
output = output.permute(0, 2, 1, 3).reshape(bsz, seqlen, -1)
|
| 367 |
+
# output = output.permute(0, 2, 1, 3).contiguous().view(bsz, seqlen, -1)
|
| 368 |
+
output = self.out_proj(output)
|
| 369 |
+
return output
|
| 370 |
+
|
| 371 |
+
class NoTPTransformerBlock(nn.Module):
|
| 372 |
+
def __init__(self, cfg, layer_id: int, multiple_of=256):
|
| 373 |
+
super().__init__()
|
| 374 |
+
|
| 375 |
+
self.n_heads = cfg.num_attention_heads
|
| 376 |
+
self.dim = cfg.hidden_size
|
| 377 |
+
self.head_dim = cfg.hidden_size // cfg.num_attention_heads
|
| 378 |
+
self.self_attn = NoTPAttention(cfg)
|
| 379 |
+
self.mlp = NoTPFeedForward(
|
| 380 |
+
cfg, dim=cfg.hidden_size, hidden_dim=cfg.ffn_hidden_size
|
| 381 |
+
)
|
| 382 |
+
self.layer_id = layer_id
|
| 383 |
+
self.layer_norm1 = torch.nn.LayerNorm(
|
| 384 |
+
cfg.hidden_size, eps=cfg.layernorm_epsilon
|
| 385 |
+
)
|
| 386 |
+
self.layer_norm2 = torch.nn.LayerNorm(
|
| 387 |
+
cfg.hidden_size, eps=cfg.layernorm_epsilon
|
| 388 |
+
)
|
| 389 |
+
|
| 390 |
+
def forward(self, x: torch.Tensor):
|
| 391 |
+
residual = self.self_attn.forward(self.layer_norm1(x))
|
| 392 |
+
h = x + residual
|
| 393 |
+
out = h + self.mlp.forward(self.layer_norm2(h))
|
| 394 |
+
return out
|
| 395 |
+
|
| 396 |
+
|
| 397 |
+
class NoTPTransformer(nn.Module):
|
| 398 |
+
def __init__(self, cfg):
|
| 399 |
+
super().__init__()
|
| 400 |
+
|
| 401 |
+
self.cfg = cfg
|
| 402 |
+
# self.recompute_list = self.cfg.get("recompute_list", [])
|
| 403 |
+
self.num_layers = cfg.num_layers # _get_num_layers(cfg)
|
| 404 |
+
|
| 405 |
+
self.layers = torch.nn.ModuleList()
|
| 406 |
+
for layer_id in range(self.num_layers):
|
| 407 |
+
self.layers.append(
|
| 408 |
+
NoTPTransformerBlock(
|
| 409 |
+
cfg,
|
| 410 |
+
layer_id + 1,
|
| 411 |
+
)
|
| 412 |
+
)
|
| 413 |
+
|
| 414 |
+
def forward(
|
| 415 |
+
self,
|
| 416 |
+
hidden_states,
|
| 417 |
+
):
|
| 418 |
+
|
| 419 |
+
for lid, layer in enumerate(self.layers):
|
| 420 |
+
# if lid in self.recompute_list:
|
| 421 |
+
# def custom(layer_id):
|
| 422 |
+
# def custom_forward(*args, **kwargs):
|
| 423 |
+
# x_ = self.layers[layer_id](*args, **kwargs)
|
| 424 |
+
# return x_
|
| 425 |
+
|
| 426 |
+
# return custom_forward
|
| 427 |
+
|
| 428 |
+
# assert hidden_states.requires_grad == True, logger.warning(
|
| 429 |
+
# "When using recalculation, the input must have grad fn"
|
| 430 |
+
# )
|
| 431 |
+
# hidden_states = tensor_parallel.checkpoint(
|
| 432 |
+
# custom(lid),
|
| 433 |
+
# False,
|
| 434 |
+
# hidden_states.contiguous()
|
| 435 |
+
# )
|
| 436 |
+
# else:
|
| 437 |
+
hidden_states = layer(hidden_states)
|
| 438 |
+
|
| 439 |
+
return hidden_states
|
| 440 |
+
|
| 441 |
+
|
| 442 |
+
# from megatron.core.tensor_parallel.layers import non_tensor_paralleled, local_dp_reduce, local_dp_scatter
|
| 443 |
+
|
| 444 |
+
class VitModel(nn.Module):
|
| 445 |
+
def __init__(
|
| 446 |
+
self,
|
| 447 |
+
cfg,
|
| 448 |
+
freeze_embed=False,
|
| 449 |
+
freeze_pre_norm=False
|
| 450 |
+
) -> None:
|
| 451 |
+
super().__init__()
|
| 452 |
+
|
| 453 |
+
self.embeddings = CLIPVisionEmbeddings(hidden_size=cfg.hidden_size, image_size=cfg.image_size, patch_size=cfg.patch_size)
|
| 454 |
+
|
| 455 |
+
if freeze_embed:
|
| 456 |
+
for name, param in self.embeddings.named_parameters():
|
| 457 |
+
param.requires_grad = False
|
| 458 |
+
|
| 459 |
+
self.transformer = NoTPTransformer(cfg=cfg)
|
| 460 |
+
|
| 461 |
+
if cfg.get("fp32norm", False):
|
| 462 |
+
logger.info("Load fp32 layernorm for ViT.")
|
| 463 |
+
self.pre_layrnorm = LayerNormfp32(
|
| 464 |
+
cfg.hidden_size,
|
| 465 |
+
eps=cfg.get("pre_layernorm_epsilon", 1e-5),
|
| 466 |
+
)
|
| 467 |
+
else:
|
| 468 |
+
self.pre_layrnorm = torch.nn.LayerNorm(
|
| 469 |
+
cfg.hidden_size,
|
| 470 |
+
eps=cfg.get("pre_layernorm_epsilon", 1e-5),
|
| 471 |
+
)
|
| 472 |
+
|
| 473 |
+
# self.pre_layrnorm = RMSNorm(
|
| 474 |
+
# cfg.hidden_size,
|
| 475 |
+
# eps=cfg.get("pre_layernorm_epsilon", 1e-5),
|
| 476 |
+
# sequence_parallel=False,
|
| 477 |
+
# use_fp32=True,
|
| 478 |
+
# use_optimus=True,
|
| 479 |
+
# )
|
| 480 |
+
|
| 481 |
+
if freeze_pre_norm:
|
| 482 |
+
for name, param in self.pre_layrnorm.named_parameters():
|
| 483 |
+
param.requires_grad = False
|
| 484 |
+
|
| 485 |
+
for p in self.parameters():
|
| 486 |
+
p.micro_dp = True
|
| 487 |
+
|
| 488 |
+
def set_input_tensor(self, input_tensor):
|
| 489 |
+
if not isinstance(input_tensor, list):
|
| 490 |
+
input_tensor = [input_tensor]
|
| 491 |
+
self.transformer.set_input_tensor(input_tensor[0])
|
| 492 |
+
|
| 493 |
+
def __str__(self) -> str:
|
| 494 |
+
return "open_clip"
|
| 495 |
+
|
| 496 |
+
def forward(
|
| 497 |
+
self,
|
| 498 |
+
x,
|
| 499 |
+
patch_embeds
|
| 500 |
+
):
|
| 501 |
+
x = self.embeddings(x, patch_embeds)
|
| 502 |
+
hidden_states = self.pre_layrnorm(x)
|
| 503 |
+
|
| 504 |
+
# hidden_states, dis = local_dp_scatter(hidden_states)
|
| 505 |
+
output = self.transformer(hidden_states)
|
| 506 |
+
|
| 507 |
+
# output = local_dp_reduce(output, dis)
|
| 508 |
+
|
| 509 |
+
return output
|
| 510 |
+
|
| 511 |
+
|
| 512 |
+
vit_model_cfg = adict(
|
| 513 |
+
num_layers=24,
|
| 514 |
+
hidden_size=1024,
|
| 515 |
+
num_heads = 16,
|
| 516 |
+
num_attention_heads=16,
|
| 517 |
+
ffn_hidden_size=4096,
|
| 518 |
+
seq_length=256,
|
| 519 |
+
max_position_embeddings=256,
|
| 520 |
+
use_flash_attn=False,
|
| 521 |
+
understand_projector_stride=2,
|
| 522 |
+
hidden_dropout = 0.0,
|
| 523 |
+
attention_dropout = 0.0,
|
| 524 |
+
no_persist_layer_norm = False,
|
| 525 |
+
layernorm_epsilon = 1e-5,
|
| 526 |
+
pre_layernorm_epsilon = 1e-5,
|
| 527 |
+
image_size = 224,
|
| 528 |
+
patch_size = 14,
|
| 529 |
+
recompute_list = []
|
| 530 |
+
)
|
| 531 |
+
|
| 532 |
+
def build_clip_l(config=None):
|
| 533 |
+
cfg = copy.deepcopy(vit_model_cfg)
|
| 534 |
+
if config is not None:
|
| 535 |
+
spec = config.vision_config['width']['clip-l-14-224']
|
| 536 |
+
cfg.update(num_layers=spec['layers'], hidden_size=spec['width'],
|
| 537 |
+
num_heads=spec['heads'], num_attention_heads=spec['heads'],
|
| 538 |
+
ffn_hidden_size=spec['width'] * 4)
|
| 539 |
+
return VitModel(
|
| 540 |
+
cfg=cfg,
|
| 541 |
+
freeze_embed=False,
|
| 542 |
+
freeze_pre_norm=False,
|
| 543 |
+
)
|
| 544 |
+
|
| 545 |
+
|
| 546 |
+
|
| 547 |
+
|
| 548 |
+
|
| 549 |
+
#=========================Sam-Vary=================================
|
| 550 |
+
|
| 551 |
+
|
| 552 |
+
def get_abs_pos_sam(abs_pos, tgt_size):
|
| 553 |
+
|
| 554 |
+
dtype = abs_pos.dtype
|
| 555 |
+
|
| 556 |
+
src_size = abs_pos.size(1)
|
| 557 |
+
|
| 558 |
+
if src_size != tgt_size:
|
| 559 |
+
old_pos_embed = abs_pos.permute(0, 3, 1, 2)
|
| 560 |
+
old_pos_embed = old_pos_embed.to(torch.float32)
|
| 561 |
+
new_pos_embed = F.interpolate(
|
| 562 |
+
old_pos_embed,
|
| 563 |
+
size=(tgt_size, tgt_size),
|
| 564 |
+
mode='bicubic',
|
| 565 |
+
antialias=True,
|
| 566 |
+
align_corners=False,
|
| 567 |
+
).to(dtype)
|
| 568 |
+
new_pos_embed = new_pos_embed.permute(0, 2, 3, 1)
|
| 569 |
+
return new_pos_embed
|
| 570 |
+
else:
|
| 571 |
+
return abs_pos
|
| 572 |
+
|
| 573 |
+
|
| 574 |
+
|
| 575 |
+
|
| 576 |
+
class MLPBlock(nn.Module):
|
| 577 |
+
def __init__(
|
| 578 |
+
self,
|
| 579 |
+
embedding_dim: int,
|
| 580 |
+
mlp_dim: int,
|
| 581 |
+
act: Type[nn.Module] = nn.GELU,
|
| 582 |
+
) -> None:
|
| 583 |
+
super().__init__()
|
| 584 |
+
self.lin1 = nn.Linear(embedding_dim, mlp_dim)
|
| 585 |
+
self.lin2 = nn.Linear(mlp_dim, embedding_dim)
|
| 586 |
+
self.act = act()
|
| 587 |
+
|
| 588 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 589 |
+
return self.lin2(self.act(self.lin1(x)))
|
| 590 |
+
|
| 591 |
+
|
| 592 |
+
# From https://github.com/facebookresearch/detectron2/blob/main/detectron2/layers/batch_norm.py # noqa
|
| 593 |
+
# Itself from https://github.com/facebookresearch/ConvNeXt/blob/d1fa8f6fef0a165b27399986cc2bdacc92777e40/models/convnext.py#L119 # noqa
|
| 594 |
+
class LayerNorm2d(nn.Module):
|
| 595 |
+
def __init__(self, num_channels: int, eps: float = 1e-6) -> None:
|
| 596 |
+
super().__init__()
|
| 597 |
+
self.weight = nn.Parameter(torch.ones(num_channels))
|
| 598 |
+
self.bias = nn.Parameter(torch.zeros(num_channels))
|
| 599 |
+
self.eps = eps
|
| 600 |
+
|
| 601 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 602 |
+
u = x.mean(1, keepdim=True)
|
| 603 |
+
s = (x - u).pow(2).mean(1, keepdim=True)
|
| 604 |
+
x = (x - u) / torch.sqrt(s + self.eps)
|
| 605 |
+
x = self.weight[:, None, None] * x + self.bias[:, None, None]
|
| 606 |
+
return x
|
| 607 |
+
|
| 608 |
+
|
| 609 |
+
# This class and its supporting functions below lightly adapted from the ViTDet backbone available at: https://github.com/facebookresearch/detectron2/blob/main/detectron2/modeling/backbone/vit.py # noqa
|
| 610 |
+
class ImageEncoderViT(nn.Module):
|
| 611 |
+
def __init__(
|
| 612 |
+
self,
|
| 613 |
+
img_size: int = 1024,
|
| 614 |
+
patch_size: int = 16,
|
| 615 |
+
in_chans: int = 3,
|
| 616 |
+
embed_dim: int = 768,
|
| 617 |
+
depth: int = 12,
|
| 618 |
+
num_heads: int = 12,
|
| 619 |
+
mlp_ratio: float = 4.0,
|
| 620 |
+
out_chans: int = 256,
|
| 621 |
+
qkv_bias: bool = True,
|
| 622 |
+
norm_layer: Type[nn.Module] = nn.LayerNorm,
|
| 623 |
+
act_layer: Type[nn.Module] = nn.GELU,
|
| 624 |
+
use_abs_pos: bool = True,
|
| 625 |
+
use_rel_pos: bool = False,
|
| 626 |
+
rel_pos_zero_init: bool = True,
|
| 627 |
+
window_size: int = 0,
|
| 628 |
+
global_attn_indexes: Tuple[int, ...] = (),
|
| 629 |
+
) -> None:
|
| 630 |
+
"""
|
| 631 |
+
Args:
|
| 632 |
+
img_size (int): Input image size.
|
| 633 |
+
patch_size (int): Patch size.
|
| 634 |
+
in_chans (int): Number of input image channels.
|
| 635 |
+
embed_dim (int): Patch embedding dimension.
|
| 636 |
+
depth (int): Depth of ViT.
|
| 637 |
+
num_heads (int): Number of attention heads in each ViT block.
|
| 638 |
+
mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
|
| 639 |
+
qkv_bias (bool): If True, add a learnable bias to query, key, value.
|
| 640 |
+
norm_layer (nn.Module): Normalization layer.
|
| 641 |
+
act_layer (nn.Module): Activation layer.
|
| 642 |
+
use_abs_pos (bool): If True, use absolute positional embeddings.
|
| 643 |
+
use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
|
| 644 |
+
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
|
| 645 |
+
window_size (int): Window size for window attention blocks.
|
| 646 |
+
global_attn_indexes (list): Indexes for blocks using global attention.
|
| 647 |
+
"""
|
| 648 |
+
super().__init__()
|
| 649 |
+
self.img_size = img_size
|
| 650 |
+
|
| 651 |
+
self.patch_embed = PatchEmbed(
|
| 652 |
+
kernel_size=(patch_size, patch_size),
|
| 653 |
+
stride=(patch_size, patch_size),
|
| 654 |
+
in_chans=in_chans,
|
| 655 |
+
embed_dim=embed_dim,
|
| 656 |
+
)
|
| 657 |
+
|
| 658 |
+
self.pos_embed: Optional[nn.Parameter] = None
|
| 659 |
+
if use_abs_pos:
|
| 660 |
+
# Initialize absolute positional embedding with pretrain image size.
|
| 661 |
+
self.pos_embed = nn.Parameter(
|
| 662 |
+
torch.zeros(1, img_size // patch_size, img_size // patch_size, embed_dim)
|
| 663 |
+
)
|
| 664 |
+
|
| 665 |
+
self.blocks = nn.ModuleList()
|
| 666 |
+
for i in range(depth):
|
| 667 |
+
block = Block(
|
| 668 |
+
dim=embed_dim,
|
| 669 |
+
num_heads=num_heads,
|
| 670 |
+
mlp_ratio=mlp_ratio,
|
| 671 |
+
qkv_bias=qkv_bias,
|
| 672 |
+
norm_layer=norm_layer,
|
| 673 |
+
act_layer=act_layer,
|
| 674 |
+
use_rel_pos=use_rel_pos,
|
| 675 |
+
rel_pos_zero_init=rel_pos_zero_init,
|
| 676 |
+
window_size=window_size if i not in global_attn_indexes else 0,
|
| 677 |
+
input_size=(img_size // patch_size, img_size // patch_size),
|
| 678 |
+
)
|
| 679 |
+
self.blocks.append(block)
|
| 680 |
+
|
| 681 |
+
self.neck = nn.Sequential(
|
| 682 |
+
nn.Conv2d(
|
| 683 |
+
embed_dim,
|
| 684 |
+
out_chans,
|
| 685 |
+
kernel_size=1,
|
| 686 |
+
bias=False,
|
| 687 |
+
),
|
| 688 |
+
LayerNorm2d(out_chans),
|
| 689 |
+
nn.Conv2d(
|
| 690 |
+
out_chans,
|
| 691 |
+
out_chans,
|
| 692 |
+
kernel_size=3,
|
| 693 |
+
padding=1,
|
| 694 |
+
bias=False,
|
| 695 |
+
),
|
| 696 |
+
LayerNorm2d(out_chans),
|
| 697 |
+
)
|
| 698 |
+
|
| 699 |
+
self.net_2 = nn.Conv2d(256, 512, kernel_size=3, stride=2, padding=1, bias=False)
|
| 700 |
+
self.net_3 = nn.Conv2d(512, 1024, kernel_size=3, stride=2, padding=1, bias=False)
|
| 701 |
+
|
| 702 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 703 |
+
x = self.patch_embed(x)
|
| 704 |
+
if self.pos_embed is not None:
|
| 705 |
+
# x = x + self.pos_embed
|
| 706 |
+
x = x + get_abs_pos_sam(self.pos_embed, x.size(1))
|
| 707 |
+
|
| 708 |
+
for blk in self.blocks:
|
| 709 |
+
x = blk(x)
|
| 710 |
+
|
| 711 |
+
x = self.neck(x.permute(0, 3, 1, 2))
|
| 712 |
+
x2 = self.net_2(x)
|
| 713 |
+
x3 = self.net_3(x2.clone())
|
| 714 |
+
|
| 715 |
+
return x3
|
| 716 |
+
|
| 717 |
+
|
| 718 |
+
class Block(nn.Module):
|
| 719 |
+
"""Transformer blocks with support of window attention and residual propagation blocks"""
|
| 720 |
+
|
| 721 |
+
def __init__(
|
| 722 |
+
self,
|
| 723 |
+
dim: int,
|
| 724 |
+
num_heads: int,
|
| 725 |
+
mlp_ratio: float = 4.0,
|
| 726 |
+
qkv_bias: bool = True,
|
| 727 |
+
norm_layer: Type[nn.Module] = nn.LayerNorm,
|
| 728 |
+
act_layer: Type[nn.Module] = nn.GELU,
|
| 729 |
+
use_rel_pos: bool = False,
|
| 730 |
+
rel_pos_zero_init: bool = True,
|
| 731 |
+
window_size: int = 0,
|
| 732 |
+
input_size: Optional[Tuple[int, int]] = None,
|
| 733 |
+
) -> None:
|
| 734 |
+
"""
|
| 735 |
+
Args:
|
| 736 |
+
dim (int): Number of input channels.
|
| 737 |
+
num_heads (int): Number of attention heads in each ViT block.
|
| 738 |
+
mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
|
| 739 |
+
qkv_bias (bool): If True, add a learnable bias to query, key, value.
|
| 740 |
+
norm_layer (nn.Module): Normalization layer.
|
| 741 |
+
act_layer (nn.Module): Activation layer.
|
| 742 |
+
use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
|
| 743 |
+
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
|
| 744 |
+
window_size (int): Window size for window attention blocks. If it equals 0, then
|
| 745 |
+
use global attention.
|
| 746 |
+
input_size (tuple(int, int) or None): Input resolution for calculating the relative
|
| 747 |
+
positional parameter size.
|
| 748 |
+
"""
|
| 749 |
+
super().__init__()
|
| 750 |
+
self.norm1 = norm_layer(dim)
|
| 751 |
+
self.attn = Attention(
|
| 752 |
+
dim,
|
| 753 |
+
num_heads=num_heads,
|
| 754 |
+
qkv_bias=qkv_bias,
|
| 755 |
+
use_rel_pos=use_rel_pos,
|
| 756 |
+
rel_pos_zero_init=rel_pos_zero_init,
|
| 757 |
+
input_size=input_size if window_size == 0 else (window_size, window_size),
|
| 758 |
+
)
|
| 759 |
+
|
| 760 |
+
self.norm2 = norm_layer(dim)
|
| 761 |
+
self.mlp = MLPBlock(embedding_dim=dim, mlp_dim=int(dim * mlp_ratio), act=act_layer)
|
| 762 |
+
|
| 763 |
+
self.window_size = window_size
|
| 764 |
+
|
| 765 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 766 |
+
shortcut = x
|
| 767 |
+
x = self.norm1(x)
|
| 768 |
+
# Window partition
|
| 769 |
+
if self.window_size > 0:
|
| 770 |
+
H, W = x.shape[1], x.shape[2]
|
| 771 |
+
x, pad_hw = window_partition(x, self.window_size)
|
| 772 |
+
|
| 773 |
+
x = self.attn(x)
|
| 774 |
+
# Reverse window partition
|
| 775 |
+
if self.window_size > 0:
|
| 776 |
+
x = window_unpartition(x, self.window_size, pad_hw, (H, W))
|
| 777 |
+
|
| 778 |
+
x = shortcut + x
|
| 779 |
+
x = x + self.mlp(self.norm2(x))
|
| 780 |
+
|
| 781 |
+
return x
|
| 782 |
+
|
| 783 |
+
|
| 784 |
+
class Attention(nn.Module):
|
| 785 |
+
"""Multi-head Attention block with relative position embeddings."""
|
| 786 |
+
|
| 787 |
+
def __init__(
|
| 788 |
+
self,
|
| 789 |
+
dim: int,
|
| 790 |
+
num_heads: int = 8,
|
| 791 |
+
qkv_bias: bool = True,
|
| 792 |
+
use_rel_pos: bool = False,
|
| 793 |
+
rel_pos_zero_init: bool = True,
|
| 794 |
+
input_size: Optional[Tuple[int, int]] = None,
|
| 795 |
+
) -> None:
|
| 796 |
+
"""
|
| 797 |
+
Args:
|
| 798 |
+
dim (int): Number of input channels.
|
| 799 |
+
num_heads (int): Number of attention heads.
|
| 800 |
+
qkv_bias (bool): If True, add a learnable bias to query, key, value.
|
| 801 |
+
rel_pos (bool): If True, add relative positional embeddings to the attention map.
|
| 802 |
+
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
|
| 803 |
+
input_size (tuple(int, int) or None): Input resolution for calculating the relative
|
| 804 |
+
positional parameter size.
|
| 805 |
+
"""
|
| 806 |
+
super().__init__()
|
| 807 |
+
self.num_heads = num_heads
|
| 808 |
+
head_dim = dim // num_heads
|
| 809 |
+
self.scale = head_dim**-0.5
|
| 810 |
+
|
| 811 |
+
self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
|
| 812 |
+
self.proj = nn.Linear(dim, dim)
|
| 813 |
+
|
| 814 |
+
self.use_rel_pos = use_rel_pos
|
| 815 |
+
if self.use_rel_pos:
|
| 816 |
+
assert (
|
| 817 |
+
input_size is not None
|
| 818 |
+
), "Input size must be provided if using relative positional encoding."
|
| 819 |
+
# initialize relative positional embeddings
|
| 820 |
+
self.rel_pos_h = nn.Parameter(torch.zeros(2 * input_size[0] - 1, head_dim))
|
| 821 |
+
self.rel_pos_w = nn.Parameter(torch.zeros(2 * input_size[1] - 1, head_dim))
|
| 822 |
+
|
| 823 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 824 |
+
B, H, W, _ = x.shape
|
| 825 |
+
# qkv with shape (3, B, nHead, H * W, C)
|
| 826 |
+
qkv = self.qkv(x).reshape(B, H * W, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)
|
| 827 |
+
# q, k, v with shape (B * nHead, H * W, C)
|
| 828 |
+
q, k, v = qkv.reshape(3, B * self.num_heads, H * W, -1).unbind(0)
|
| 829 |
+
|
| 830 |
+
rel_h, rel_w = None, None
|
| 831 |
+
if self.use_rel_pos:
|
| 832 |
+
rel_h, rel_w = add_decomposed_rel_pos(q, self.rel_pos_h, self.rel_pos_w, (H, W), (H, W))
|
| 833 |
+
|
| 834 |
+
q = q.view(B, self.num_heads, H * W, -1)
|
| 835 |
+
k = k.view(B, self.num_heads, H * W, -1)
|
| 836 |
+
v = v.view(B, self.num_heads, H * W, -1)
|
| 837 |
+
|
| 838 |
+
if self.use_rel_pos:
|
| 839 |
+
rel_h = rel_h.view(B, self.num_heads, rel_h.size(1), rel_h.size(2), rel_h.size(3))
|
| 840 |
+
rel_w = rel_w.view(B, self.num_heads, rel_w.size(1), rel_w.size(2), rel_w.size(3))
|
| 841 |
+
attn_bias = (rel_h + rel_w).view(B, self.num_heads, rel_h.size(2), rel_h.size(3) * rel_w.size(4))
|
| 842 |
+
x = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=attn_bias)
|
| 843 |
+
# x = _attention_rel_h_rel_w(q, k, v, rel_h, rel_w)
|
| 844 |
+
else:
|
| 845 |
+
x = torch.nn.functional.scaled_dot_product_attention(q, k, v)
|
| 846 |
+
|
| 847 |
+
x = x.view(B, self.num_heads, H, W, -1).permute(0, 2, 3, 1, 4).reshape(B, H, W, -1)
|
| 848 |
+
|
| 849 |
+
x = self.proj(x)
|
| 850 |
+
|
| 851 |
+
return x
|
| 852 |
+
|
| 853 |
+
|
| 854 |
+
def window_partition(x: torch.Tensor, window_size: int) -> Tuple[torch.Tensor, Tuple[int, int]]:
|
| 855 |
+
"""
|
| 856 |
+
Partition into non-overlapping windows with padding if needed.
|
| 857 |
+
Args:
|
| 858 |
+
x (tensor): input tokens with [B, H, W, C].
|
| 859 |
+
window_size (int): window size.
|
| 860 |
+
|
| 861 |
+
Returns:
|
| 862 |
+
windows: windows after partition with [B * num_windows, window_size, window_size, C].
|
| 863 |
+
(Hp, Wp): padded height and width before partition
|
| 864 |
+
"""
|
| 865 |
+
B, H, W, C = x.shape
|
| 866 |
+
|
| 867 |
+
pad_h = (window_size - H % window_size) % window_size
|
| 868 |
+
pad_w = (window_size - W % window_size) % window_size
|
| 869 |
+
if pad_h > 0 or pad_w > 0:
|
| 870 |
+
x = F.pad(x, (0, 0, 0, pad_w, 0, pad_h))
|
| 871 |
+
Hp, Wp = H + pad_h, W + pad_w
|
| 872 |
+
|
| 873 |
+
x = x.view(B, Hp // window_size, window_size, Wp // window_size, window_size, C)
|
| 874 |
+
windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)
|
| 875 |
+
return windows, (Hp, Wp)
|
| 876 |
+
|
| 877 |
+
|
| 878 |
+
def window_unpartition(
|
| 879 |
+
windows: torch.Tensor, window_size: int, pad_hw: Tuple[int, int], hw: Tuple[int, int]
|
| 880 |
+
) -> torch.Tensor:
|
| 881 |
+
"""
|
| 882 |
+
Window unpartition into original sequences and removing padding.
|
| 883 |
+
Args:
|
| 884 |
+
windows (tensor): input tokens with [B * num_windows, window_size, window_size, C].
|
| 885 |
+
window_size (int): window size.
|
| 886 |
+
pad_hw (Tuple): padded height and width (Hp, Wp).
|
| 887 |
+
hw (Tuple): original height and width (H, W) before padding.
|
| 888 |
+
|
| 889 |
+
Returns:
|
| 890 |
+
x: unpartitioned sequences with [B, H, W, C].
|
| 891 |
+
"""
|
| 892 |
+
Hp, Wp = pad_hw
|
| 893 |
+
H, W = hw
|
| 894 |
+
B = windows.shape[0] // (Hp * Wp // window_size // window_size)
|
| 895 |
+
x = windows.view(B, Hp // window_size, Wp // window_size, window_size, window_size, -1)
|
| 896 |
+
x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, Hp, Wp, -1)
|
| 897 |
+
|
| 898 |
+
if Hp > H or Wp > W:
|
| 899 |
+
x = x[:, :H, :W, :].contiguous()
|
| 900 |
+
return x
|
| 901 |
+
|
| 902 |
+
|
| 903 |
+
def get_rel_pos(q_size: int, k_size: int, rel_pos: torch.Tensor) -> torch.Tensor:
|
| 904 |
+
"""
|
| 905 |
+
Get relative positional embeddings according to the relative positions of
|
| 906 |
+
query and key sizes.
|
| 907 |
+
Args:
|
| 908 |
+
q_size (int): size of query q.
|
| 909 |
+
k_size (int): size of key k.
|
| 910 |
+
rel_pos (Tensor): relative position embeddings (L, C).
|
| 911 |
+
|
| 912 |
+
Returns:
|
| 913 |
+
Extracted positional embeddings according to relative positions.
|
| 914 |
+
"""
|
| 915 |
+
max_rel_dist = int(2 * max(q_size, k_size) - 1)
|
| 916 |
+
# Interpolate rel pos if needed.
|
| 917 |
+
if rel_pos.shape[0] != max_rel_dist:
|
| 918 |
+
# Interpolate rel pos.
|
| 919 |
+
dtype = rel_pos.dtype
|
| 920 |
+
rel_pos = rel_pos.to(torch.float32)
|
| 921 |
+
rel_pos_resized = F.interpolate(
|
| 922 |
+
rel_pos.reshape(1, rel_pos.shape[0], -1).permute(0, 2, 1),
|
| 923 |
+
size=max_rel_dist,
|
| 924 |
+
mode="linear",
|
| 925 |
+
).to(dtype)
|
| 926 |
+
rel_pos_resized = rel_pos_resized.reshape(-1, max_rel_dist).permute(1, 0)
|
| 927 |
+
else:
|
| 928 |
+
rel_pos_resized = rel_pos
|
| 929 |
+
|
| 930 |
+
# Scale the coords with short length if shapes for q and k are different.
|
| 931 |
+
q_coords = torch.arange(q_size, device=rel_pos.device)[:, None] * max(k_size / q_size, 1.0)
|
| 932 |
+
k_coords = torch.arange(k_size, device=rel_pos.device)[None, :] * max(q_size / k_size, 1.0)
|
| 933 |
+
relative_coords = (q_coords - k_coords) + (k_size - 1) * max(q_size / k_size, 1.0)
|
| 934 |
+
|
| 935 |
+
return rel_pos_resized[relative_coords.long()]
|
| 936 |
+
|
| 937 |
+
|
| 938 |
+
def add_decomposed_rel_pos(
|
| 939 |
+
q: torch.Tensor,
|
| 940 |
+
rel_pos_h: torch.Tensor,
|
| 941 |
+
rel_pos_w: torch.Tensor,
|
| 942 |
+
q_size: Tuple[int, int],
|
| 943 |
+
k_size: Tuple[int, int],
|
| 944 |
+
) -> torch.Tensor:
|
| 945 |
+
"""
|
| 946 |
+
Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`.
|
| 947 |
+
https://github.com/facebookresearch/mvit/blob/19786631e330df9f3622e5402b4a419a263a2c80/mvit/models/attention.py # noqa B950
|
| 948 |
+
Args:
|
| 949 |
+
q (Tensor): query q in the attention layer with shape (B, q_h * q_w, C).
|
| 950 |
+
rel_pos_h (Tensor): relative position embeddings (Lh, C) for height axis.
|
| 951 |
+
rel_pos_w (Tensor): relative position embeddings (Lw, C) for width axis.
|
| 952 |
+
q_size (Tuple): spatial sequence size of query q with (q_h, q_w).
|
| 953 |
+
k_size (Tuple): spatial sequence size of key k with (k_h, k_w).
|
| 954 |
+
|
| 955 |
+
Returns:
|
| 956 |
+
attn (Tensor): attention map with added relative positional embeddings.
|
| 957 |
+
"""
|
| 958 |
+
q_h, q_w = q_size
|
| 959 |
+
k_h, k_w = k_size
|
| 960 |
+
Rh = get_rel_pos(q_h, k_h, rel_pos_h)
|
| 961 |
+
Rw = get_rel_pos(q_w, k_w, rel_pos_w)
|
| 962 |
+
|
| 963 |
+
B, _, dim = q.shape
|
| 964 |
+
r_q = q.reshape(B, q_h, q_w, dim)
|
| 965 |
+
rel_h = torch.einsum("bhwc,hkc->bhwk", r_q, Rh)
|
| 966 |
+
rel_w = torch.einsum("bhwc,wkc->bhwk", r_q, Rw)
|
| 967 |
+
rel_h = rel_h.unsqueeze(-1)
|
| 968 |
+
rel_w = rel_w.unsqueeze(-2)
|
| 969 |
+
rel_h = rel_h.reshape(B, q_h * q_w, k_h, 1)
|
| 970 |
+
rel_w = rel_w.reshape(B, q_h * q_w, 1, k_w)
|
| 971 |
+
|
| 972 |
+
return rel_h, rel_w
|
| 973 |
+
|
| 974 |
+
|
| 975 |
+
class PatchEmbed(nn.Module):
|
| 976 |
+
"""
|
| 977 |
+
Image to Patch Embedding.
|
| 978 |
+
"""
|
| 979 |
+
|
| 980 |
+
def __init__(
|
| 981 |
+
self,
|
| 982 |
+
kernel_size: Tuple[int, int] = (16, 16),
|
| 983 |
+
stride: Tuple[int, int] = (16, 16),
|
| 984 |
+
padding: Tuple[int, int] = (0, 0),
|
| 985 |
+
in_chans: int = 3,
|
| 986 |
+
embed_dim: int = 768,
|
| 987 |
+
) -> None:
|
| 988 |
+
"""
|
| 989 |
+
Args:
|
| 990 |
+
kernel_size (Tuple): kernel size of the projection layer.
|
| 991 |
+
stride (Tuple): stride of the projection layer.
|
| 992 |
+
padding (Tuple): padding size of the projection layer.
|
| 993 |
+
in_chans (int): Number of input image channels.
|
| 994 |
+
embed_dim (int): Patch embedding dimension.
|
| 995 |
+
"""
|
| 996 |
+
super().__init__()
|
| 997 |
+
|
| 998 |
+
self.proj = nn.Conv2d(
|
| 999 |
+
in_chans, embed_dim, kernel_size=kernel_size, stride=stride, padding=padding
|
| 1000 |
+
)
|
| 1001 |
+
|
| 1002 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 1003 |
+
x = self.proj(x)
|
| 1004 |
+
# B C H W -> B H W C
|
| 1005 |
+
x = x.permute(0, 2, 3, 1)
|
| 1006 |
+
return x
|
| 1007 |
+
|
| 1008 |
+
|
| 1009 |
+
def build_sam_vit_b(checkpoint=None, config=None):
|
| 1010 |
+
if config is not None:
|
| 1011 |
+
spec = config.vision_config['width']['sam_vit_b']
|
| 1012 |
+
encoder = ImageEncoderViT(embed_dim=spec['width'], depth=spec['layers'],
|
| 1013 |
+
num_heads=spec['heads'], global_attn_indexes=spec['global_attn_indexes'],
|
| 1014 |
+
out_chans=spec['neck_channels'], window_size=14, use_rel_pos=True,
|
| 1015 |
+
norm_layer=partial(torch.nn.LayerNorm, eps=1e-6))
|
| 1016 |
+
c1, c2 = spec['downsample_channels']
|
| 1017 |
+
encoder.net_2 = nn.Conv2d(spec['neck_channels'], c1, 3, stride=2, padding=1, bias=False)
|
| 1018 |
+
encoder.net_3 = nn.Conv2d(c1, c2, 3, stride=2, padding=1, bias=False)
|
| 1019 |
+
return encoder
|
| 1020 |
+
return _build_sam(
|
| 1021 |
+
encoder_embed_dim=768,
|
| 1022 |
+
encoder_depth=12,
|
| 1023 |
+
encoder_num_heads=12,
|
| 1024 |
+
encoder_global_attn_indexes=[2, 5, 8, 11],
|
| 1025 |
+
checkpoint=checkpoint,
|
| 1026 |
+
)
|
| 1027 |
+
|
| 1028 |
+
def build_sam_fast_vit_b(checkpoint=None, compile_mode='max-autotune', dtype=torch.bfloat16):
|
| 1029 |
+
image_encoder = build_sam_vit_b(checkpoint).eval().to(dtype)
|
| 1030 |
+
# sam = _apply_eval_dtype_sam(sam, dtype)
|
| 1031 |
+
image_encoder = torch.compile(image_encoder, mode=compile_mode)
|
| 1032 |
+
return image_encoder
|
| 1033 |
+
|
| 1034 |
+
|
| 1035 |
+
def _build_sam(
|
| 1036 |
+
encoder_embed_dim,
|
| 1037 |
+
encoder_depth,
|
| 1038 |
+
encoder_num_heads,
|
| 1039 |
+
encoder_global_attn_indexes,
|
| 1040 |
+
checkpoint=None,
|
| 1041 |
+
):
|
| 1042 |
+
prompt_embed_dim = 256
|
| 1043 |
+
image_size = 1024
|
| 1044 |
+
vit_patch_size = 16
|
| 1045 |
+
image_embedding_size = image_size // vit_patch_size
|
| 1046 |
+
image_encoder=ImageEncoderViT(
|
| 1047 |
+
depth=encoder_depth,
|
| 1048 |
+
embed_dim=encoder_embed_dim,
|
| 1049 |
+
img_size=image_size,
|
| 1050 |
+
mlp_ratio=4,
|
| 1051 |
+
norm_layer=partial(torch.nn.LayerNorm, eps=1e-6),
|
| 1052 |
+
num_heads=encoder_num_heads,
|
| 1053 |
+
patch_size=vit_patch_size,
|
| 1054 |
+
qkv_bias=True,
|
| 1055 |
+
use_rel_pos=True,
|
| 1056 |
+
global_attn_indexes=encoder_global_attn_indexes,
|
| 1057 |
+
window_size=14,
|
| 1058 |
+
out_chans=prompt_embed_dim,
|
| 1059 |
+
)
|
| 1060 |
+
image_encoder.eval()
|
| 1061 |
+
if checkpoint is not None:
|
| 1062 |
+
# with open(checkpoint, "rb") as f:
|
| 1063 |
+
state_dict = torch.load(checkpoint)
|
| 1064 |
+
# print(state_dict.keys())
|
| 1065 |
+
# for key in state_dict:
|
| 1066 |
+
# image_encoder.load_state_dict({k[14:]: v for k, v in state_dict.items() if 'image_encoder' in k}, strict=False)
|
| 1067 |
+
# ocr-anyting
|
| 1068 |
+
# image_encoder.load_state_dict(state_dict, strict=True)
|
| 1069 |
+
# tob
|
| 1070 |
+
image_encoder.load_state_dict({k[30:]: v for k, v in state_dict.items() if 'vision_tower_high' in k}, strict=True)
|
| 1071 |
+
print(checkpoint)
|
| 1072 |
+
return image_encoder
|
generation_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 0,
|
| 4 |
+
"eos_token_id": 1,
|
| 5 |
+
"transformers_version": "4.52.1"
|
| 6 |
+
}
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:35106c601d5693dc748814318feefbf609ea8a15b00e3ade28dd8296551ebd7f
|
| 3 |
+
size 18123048
|
modeling_deepseekv2.py
ADDED
|
@@ -0,0 +1,531 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""DASH overlay on top of the page-isolated (page2d) base modeling.
|
| 2 |
+
|
| 3 |
+
Thin monkey-patch: `_sparse_base_impl.py` carries the full base implementation; this file
|
| 4 |
+
only swaps the **prefill** path of SlidingWindowLlamaAttention for
|
| 5 |
+
dash_attention.DashAttention (entmax soft prior + log(prior) bias, gradients reaching the
|
| 6 |
+
router straight from the LM loss, no KL teacher) when
|
| 7 |
+
`sparse_attention_config.training_stage == "dash"`. Decode, the non-dash stages and all
|
| 8 |
+
other classes keep using the base implementation.
|
| 9 |
+
|
| 10 |
+
Three details the dash path relies on -- page isolation degrades silently if any of them
|
| 11 |
+
is dropped:
|
| 12 |
+
|
| 13 |
+
1. The main-branch rope goes through `self._main_rope(...)` rather than
|
| 14 |
+
`self.rotary_emb(...)`, otherwise sparse_main_rope_mode=page2d is bypassed and the main
|
| 15 |
+
branch falls back to 1D.
|
| 16 |
+
2. `self.indexer` is **not** set to None: `_main_rope_coords` returns None without an
|
| 17 |
+
indexer (main branch falls back to 1D), and every page/rope helper
|
| 18 |
+
(_build_page_aware_ids / _build_page_rope_pos / rotary_emb_2d /
|
| 19 |
+
_build_kl_candidate_mask) lives on the indexer.
|
| 20 |
+
3. Router scoring uses the indexer's own 2-axis rope (sparse_rope_*: one page pair at
|
| 21 |
+
theta=256 with the page index taken mod 2, plus 63 token pairs at theta=1e4, resampled)
|
| 22 |
+
together with a page-aware PER-QUERY candidate domain [B,S,nblk].
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
import copy
|
| 26 |
+
import math
|
| 27 |
+
import os
|
| 28 |
+
|
| 29 |
+
import torch
|
| 30 |
+
import torch.nn.functional as F
|
| 31 |
+
|
| 32 |
+
if os.environ.get("DASH_DEBUG_ANOMALY") == "1":
|
| 33 |
+
torch.autograd.set_detect_anomaly(True)
|
| 34 |
+
print("[DASH] torch.autograd anomaly detection ENABLED", flush=True)
|
| 35 |
+
|
| 36 |
+
# Re-export the ENTIRE base namespace (including underscore helpers) so that
|
| 37 |
+
# modeling_unlimitedocr.py's `from .modeling_deepseekv2 import <name>` keeps working.
|
| 38 |
+
from . import _sparse_base_impl as _base # noqa: E402
|
| 39 |
+
|
| 40 |
+
globals().update({k: v for k, v in vars(_base).items() if not k.startswith("__")})
|
| 41 |
+
|
| 42 |
+
from ._sparse_base_impl import ( # noqa: E402
|
| 43 |
+
DeepseekV2PreTrainedModel as _BasePreTrained,
|
| 44 |
+
SlidingWindowLlamaAttention as _BaseAttn,
|
| 45 |
+
_llama_apply_rotary_pos_emb,
|
| 46 |
+
_llama_repeat_kv,
|
| 47 |
+
)
|
| 48 |
+
from .dash_attention import DashAttention # noqa: E402
|
| 49 |
+
from .dash_attention import entmax15 as _entmax15 # noqa: E402
|
| 50 |
+
from .dash_attention import _SCORE_NEG, _ACT_CLAMP, _MASK_NEG # noqa: E402
|
| 51 |
+
|
| 52 |
+
_orig_init = _BaseAttn.__init__
|
| 53 |
+
_orig_forward = _BaseAttn.forward
|
| 54 |
+
_DASH_LOGGED = False
|
| 55 |
+
_IDX_ROPE_WARNED = False
|
| 56 |
+
# id(sparse_attention_config) -> deep copy taken BEFORE DashAttention.__init__ rewrites it
|
| 57 |
+
# (_persist_router_cfg writes the live process state -- k_start=64, env DASH_STAGE0 default
|
| 58 |
+
# "indexer" -- over dash_topk_blocks / dash_stage0 / ...). Inference reads the snapshot.
|
| 59 |
+
_CKPT_SP = {}
|
| 60 |
+
# Inference switch, toggled by UnlimitedOCRForCausalLM.infer (see configure_dash_inference).
|
| 61 |
+
# When on, DASH layers route exactly like vLLM's `unlimited_page2d_dash` plugin
|
| 62 |
+
# (dash_router.DashRouterMath.route_and_select: top-M entmax window + hard top-K read set)
|
| 63 |
+
# in prefill AND decode; decode no longer falls back to dense full-cache attention.
|
| 64 |
+
_INFER = {"on": False}
|
| 65 |
+
_ENTMAX_NEG = -1.0e4
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def _patched_init(self, config, layer_idx):
|
| 69 |
+
sp = getattr(config, "sparse_attention_config", None)
|
| 70 |
+
is_dash = isinstance(sp, dict) and sp.get("training_stage") == "dash"
|
| 71 |
+
if is_dash and id(sp) not in _CKPT_SP:
|
| 72 |
+
_CKPT_SP[id(sp)] = copy.deepcopy(sp)
|
| 73 |
+
if is_dash:
|
| 74 |
+
_saved_stage = sp.get("training_stage")
|
| 75 |
+
sp["training_stage"] = "warmup"
|
| 76 |
+
_orig_init(self, config, layer_idx)
|
| 77 |
+
if is_dash:
|
| 78 |
+
sp["training_stage"] = _saved_stage
|
| 79 |
+
|
| 80 |
+
self.dash_router = None
|
| 81 |
+
if is_dash and self.indexer is not None:
|
| 82 |
+
self.dash_router = DashAttention(config)
|
| 83 |
+
global _DASH_LOGGED
|
| 84 |
+
if not _DASH_LOGGED:
|
| 85 |
+
_DASH_LOGGED = True
|
| 86 |
+
print(f"[DASH] differentiable DASH attention active "
|
| 87 |
+
f"(entmax router, block_size={self.dash_router.block_size}, "
|
| 88 |
+
f"local_window={self.dash_router.local_window}, "
|
| 89 |
+
f"index_dim={self.dash_router.index_dim}, "
|
| 90 |
+
f"main_rope={self.main_rope_mode}, "
|
| 91 |
+
f"idx_rope={self.indexer.rope_mode}, "
|
| 92 |
+
f"page_bucket={self.indexer.image_bucket_constraint}); "
|
| 93 |
+
f"first dash layer_idx={layer_idx}", flush=True)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def _dash_page_ids(self, idx, src_ids, key_length):
|
| 97 |
+
"""Page-aware ids, built with the same calls as the base prefill path."""
|
| 98 |
+
if idx.image_bucket_constraint:
|
| 99 |
+
itm, ipi, qpi = idx._build_page_aware_ids(
|
| 100 |
+
src_ids, idx.image_token_id, idx.page_token_id, key_length)
|
| 101 |
+
else:
|
| 102 |
+
itm = idx._build_flat_image_token_mask(src_ids, idx.image_token_id, key_length)
|
| 103 |
+
ipi = qpi = None
|
| 104 |
+
idx._log_image_mask(itm, src_ids, key_length)
|
| 105 |
+
return itm, ipi, qpi
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _dash_block_cand(self, idx, router, pos_ids, itm, ipi, qpi, nblk, bsz, q_len):
|
| 109 |
+
"""PER-QUERY candidate domain [B,S,nblk], reusing the indexer's
|
| 110 |
+
_build_kl_candidate_mask (same-page image bucket + per-query fallback). Built in
|
| 111 |
+
q_chunk slices so the [B,S,nblk,block_size] intermediate is never materialised at
|
| 112 |
+
once (32k x 1024 x 32 bool is roughly 1 GiB)."""
|
| 113 |
+
if itm is None and (ipi is None or qpi is None):
|
| 114 |
+
return None
|
| 115 |
+
step = max(1, int(router.q_chunk))
|
| 116 |
+
out = []
|
| 117 |
+
for c0 in range(0, q_len, step):
|
| 118 |
+
c1 = min(c0 + step, q_len)
|
| 119 |
+
cm = idx._build_kl_candidate_mask(
|
| 120 |
+
pos_ids[:, c0:c1], nblk, itm, bsz, c1 - c0,
|
| 121 |
+
image_token_page_ids=ipi, query_token_page_ids=qpi) # [B,1,C,nblk]
|
| 122 |
+
out.append(cm[:, 0])
|
| 123 |
+
return torch.cat(out, dim=1) # [B,S,nblk]
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def _dash_indexer_rope(self, idx, router, x, pos_ids, coords, main_pe):
|
| 127 |
+
"""cos/sin used for router scoring: the indexer's 2-axis rope (sparse_rope_*) when
|
| 128 |
+
the coordinates are available, otherwise the indexer's 1D rope. With
|
| 129 |
+
index_dim < head_dim the two axes cannot be sliced (cat(ang, ang) + rotate_half would
|
| 130 |
+
mix the phases of the two axes), so 1D is used instead."""
|
| 131 |
+
global _IDX_ROPE_WARNED
|
| 132 |
+
if coords is not None and getattr(idx, "rotary_emb_2d", None) is not None:
|
| 133 |
+
if router.index_dim == idx.head_dim:
|
| 134 |
+
return idx.rotary_emb_2d(x, coords[0], coords[1])
|
| 135 |
+
if not _IDX_ROPE_WARNED:
|
| 136 |
+
_IDX_ROPE_WARNED = True
|
| 137 |
+
print(f"[PageRope][WARN] sparse_index_dim={router.index_dim} != "
|
| 138 |
+
f"head_dim={idx.head_dim}: the 2-axis cos/sin cannot be sliced per "
|
| 139 |
+
f"frequency pair, router scoring falls back to 1D RoPE "
|
| 140 |
+
f"(the main branch is unaffected)", flush=True)
|
| 141 |
+
if getattr(idx, "rotary_emb_1d", None) is not None:
|
| 142 |
+
return idx.rotary_emb_1d(x, pos_ids)
|
| 143 |
+
return main_pe
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def _dash_attn_forward(self, hidden_states, attention_mask, position_ids, past_kv):
|
| 147 |
+
idx = self.indexer
|
| 148 |
+
router = self.dash_router
|
| 149 |
+
num_heads = self.config.num_attention_heads
|
| 150 |
+
num_kv_heads = self.config.num_key_value_heads
|
| 151 |
+
head_dim = self.head_dim
|
| 152 |
+
num_kv_groups = self.num_key_value_groups
|
| 153 |
+
bsz, q_len, _ = hidden_states.size()
|
| 154 |
+
|
| 155 |
+
pos_ids = position_ids
|
| 156 |
+
if _base._pk.is_packed(pos_ids):
|
| 157 |
+
raise RuntimeError(
|
| 158 |
+
"[DASH] packing rows are not supported by the dash path "
|
| 159 |
+
"(fused e2e kernel requires attn_mask=None, and page rope would run the "
|
| 160 |
+
"<PAGE> cumsum across sample boundaries). Disable packing.")
|
| 161 |
+
|
| 162 |
+
q = self.q_proj(hidden_states).view(bsz, q_len, num_heads, head_dim).transpose(1, 2)
|
| 163 |
+
key_states = self.k_proj(hidden_states).view(bsz, q_len, num_kv_heads, head_dim).transpose(1, 2)
|
| 164 |
+
value_states = self.v_proj(hidden_states).view(bsz, q_len, num_kv_heads, head_dim).transpose(1, 2)
|
| 165 |
+
|
| 166 |
+
src_ids = getattr(self, '_sparse_source_input_ids', None)
|
| 167 |
+
itm, ipi, qpi = _dash_page_ids(self, idx, src_ids, q_len)
|
| 168 |
+
|
| 169 |
+
coords = None
|
| 170 |
+
if idx.rope_mode == "page2d":
|
| 171 |
+
coords = self._main_rope_pos_2d
|
| 172 |
+
if coords is not None and coords[0].shape[-1] != q_len:
|
| 173 |
+
coords = None
|
| 174 |
+
if coords is None:
|
| 175 |
+
coords = idx._build_page_rope_pos(src_ids, q_len)
|
| 176 |
+
idx._rope_pos_2d = coords
|
| 177 |
+
if self.main_rope_mode == "page2d" and coords is not None:
|
| 178 |
+
self._main_rope_pos_2d = coords
|
| 179 |
+
cos, sin = self._main_rope(value_states, pos_ids, past_kv, q_len)
|
| 180 |
+
else:
|
| 181 |
+
self._main_rope_pos_2d = None
|
| 182 |
+
cos, sin = self.rotary_emb(value_states, pos_ids)
|
| 183 |
+
q, key_states = _llama_apply_rotary_pos_emb(q, key_states, cos, sin)
|
| 184 |
+
|
| 185 |
+
if past_kv is not None:
|
| 186 |
+
key_states, value_states = past_kv.update(key_states, value_states, self.layer_idx)
|
| 187 |
+
k = _llama_repeat_kv(key_states, num_kv_groups)
|
| 188 |
+
v = _llama_repeat_kv(value_states, num_kv_groups)
|
| 189 |
+
idx_pe = _dash_indexer_rope(self, idx, router, value_states, pos_ids, coords, (cos, sin))
|
| 190 |
+
|
| 191 |
+
nblk = -(-k.shape[2] // router.block_size)
|
| 192 |
+
block_cand = _dash_block_cand(self, idx, router, pos_ids, itm, ipi, qpi,
|
| 193 |
+
nblk, bsz, q_len)
|
| 194 |
+
|
| 195 |
+
router_mask = attention_mask
|
| 196 |
+
if router_mask is not None and bool((router_mask[:, :, -1, :] > -1.0).all()):
|
| 197 |
+
router_mask = None
|
| 198 |
+
|
| 199 |
+
if _INFER["on"]:
|
| 200 |
+
attn_output = _infer_prefill_attend(self, router, hidden_states, q, k, v, idx_pe,
|
| 201 |
+
pos_ids, block_cand, router_mask, past_kv)
|
| 202 |
+
else:
|
| 203 |
+
attn_output = router(hidden_states, q, k, v,
|
| 204 |
+
position_embeddings=idx_pe,
|
| 205 |
+
position_ids=pos_ids,
|
| 206 |
+
attention_mask=router_mask,
|
| 207 |
+
image_block_mask=None,
|
| 208 |
+
block_cand=block_cand,
|
| 209 |
+
page_ids=(ipi, qpi))
|
| 210 |
+
attn_output = attn_output.transpose(1, 2).contiguous().reshape(bsz, q_len, -1)
|
| 211 |
+
attn_output = self.o_proj(attn_output)
|
| 212 |
+
|
| 213 |
+
return attn_output, None, past_kv, None
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def _patched_forward(self, *args, **kwargs):
|
| 217 |
+
if self.dash_router is not None:
|
| 218 |
+
def _get(name, idx_, default=None):
|
| 219 |
+
if name in kwargs:
|
| 220 |
+
return kwargs[name]
|
| 221 |
+
if len(args) > idx_:
|
| 222 |
+
return args[idx_]
|
| 223 |
+
return default
|
| 224 |
+
|
| 225 |
+
hidden_states = _get('hidden_states', 0)
|
| 226 |
+
attention_mask = _get('attention_mask', 1)
|
| 227 |
+
position_ids = _get('position_ids', 2)
|
| 228 |
+
past_kv = _get('past_key_value', 3)
|
| 229 |
+
if past_kv is None:
|
| 230 |
+
past_kv = kwargs.get('past_key_values', None)
|
| 231 |
+
|
| 232 |
+
bsz, q_len, _ = hidden_states.size()
|
| 233 |
+
if q_len > 1:
|
| 234 |
+
if position_ids is None:
|
| 235 |
+
position_ids = torch.arange(
|
| 236 |
+
q_len, device=hidden_states.device).unsqueeze(0).expand(bsz, -1)
|
| 237 |
+
return _dash_attn_forward(self, hidden_states, attention_mask,
|
| 238 |
+
position_ids, past_kv)
|
| 239 |
+
if (_INFER["on"] and past_kv is not None
|
| 240 |
+
and self.layer_idx in getattr(past_kv, "_dash_idx_k", {})):
|
| 241 |
+
return _dash_decode_forward(self, hidden_states, position_ids, past_kv)
|
| 242 |
+
|
| 243 |
+
return _orig_forward(self, *args, **kwargs)
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
_BaseAttn.__init__ = _patched_init
|
| 247 |
+
_BaseAttn.forward = _patched_forward
|
| 248 |
+
|
| 249 |
+
# ---------------------------------------------------------------------------
|
| 250 |
+
# ---------------------------------------------------------------------------
|
| 251 |
+
_orig_init_weights = _BasePreTrained._init_weights
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def _patched_init_weights(self, module):
|
| 255 |
+
_orig_init_weights(self, module)
|
| 256 |
+
if isinstance(module, DashAttention):
|
| 257 |
+
module.head_cls.data.zero_()
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
_BasePreTrained._init_weights = _patched_init_weights
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
# ===========================================================================
|
| 264 |
+
# Inference path -- parity with vLLM's `unlimited_page2d_dash` plugin
|
| 265 |
+
# (plugin_page2d_dash/unlimited_page2d_dash: dash_router.py + dash_attention_layer.py).
|
| 266 |
+
# Only active between configure_dash_inference(model, True) / (model, False), i.e. inside
|
| 267 |
+
# UnlimitedOCRForCausalLM.infer; training never enters these functions.
|
| 268 |
+
#
|
| 269 |
+
# vs. the training forward (DashAttention.forward) the inference router differs in:
|
| 270 |
+
# * routing params come from the checkpoint snapshot (_CKPT_SP): dash_topk_blocks (hard
|
| 271 |
+
# top-K read budget), dash_stage0, per-layer dash_scale_per_layer / dash_floor_per_layer
|
| 272 |
+
# -- the live config dict has been overwritten by _persist_router_cfg at construction;
|
| 273 |
+
# * entmax runs on the top-M (DASH_INFER_TOPM, deploy default 16) scores, then the read
|
| 274 |
+
# set is the first K candidate slots (DashRouterMath.route_and_select, budget=topk);
|
| 275 |
+
# * decode (q_len == 1) is ALSO block-sparse (vLLM DASH_DECODE_DENSE=0): head_cls block
|
| 276 |
+
# summaries over the cached router keys, page-aware candidate mask, bias-aware attend
|
| 277 |
+
# over the selected blocks + the token-level local window.
|
| 278 |
+
# ===========================================================================
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
def _sched_slot(sp, layer_idx):
|
| 282 |
+
freq = sp.get("sparse_attention_freq")
|
| 283 |
+
if not isinstance(freq, (list, tuple)) or layer_idx >= len(freq):
|
| 284 |
+
return int(layer_idx)
|
| 285 |
+
return sum(1 for f in freq[:layer_idx] if f)
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
def _per_layer_val(sp, key, layer_idx):
|
| 289 |
+
arr = sp.get(key)
|
| 290 |
+
if not isinstance(arr, (list, tuple)) or not arr:
|
| 291 |
+
return None
|
| 292 |
+
slot = _sched_slot(sp, int(layer_idx))
|
| 293 |
+
return float(arr[slot]) if 0 <= slot < len(arr) else None
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def configure_dash_inference(decoder, on, topm=16, page_scope_prefill=True, state=None):
|
| 297 |
+
"""Switch the vLLM-parity DASH inference path on/off for ``decoder`` (DeepseekV2Model).
|
| 298 |
+
|
| 299 |
+
on=True returns a state dict; pass it back with on=False to restore.
|
| 300 |
+
"""
|
| 301 |
+
if not on:
|
| 302 |
+
if state is not None:
|
| 303 |
+
_base._PAGE_SCOPE_PREFILL = state["page_scope_prefill"]
|
| 304 |
+
for attn, stage0 in state["stage0"]:
|
| 305 |
+
attn.dash_router.stage0 = stage0
|
| 306 |
+
_INFER["on"] = False
|
| 307 |
+
return None
|
| 308 |
+
state = {"page_scope_prefill": _base._PAGE_SCOPE_PREFILL, "stage0": []}
|
| 309 |
+
cfg = decoder.config
|
| 310 |
+
live = getattr(cfg, "sparse_attention_config", None) or {}
|
| 311 |
+
sp = _CKPT_SP.get(id(live), live)
|
| 312 |
+
stage0 = str(os.environ.get("DASH_STAGE0") or sp.get("dash_stage0") or "indexer").lower()
|
| 313 |
+
budget = sp.get("dash_topk_blocks") or sp.get("dash_image_fallback_remote") or 12
|
| 314 |
+
for layer in decoder.layers:
|
| 315 |
+
attn = getattr(layer, "self_attn", None)
|
| 316 |
+
router = getattr(attn, "dash_router", None)
|
| 317 |
+
if router is None:
|
| 318 |
+
continue
|
| 319 |
+
li = attn.layer_idx
|
| 320 |
+
scale = _per_layer_val(sp, "dash_scale_per_layer", li)
|
| 321 |
+
floor = _per_layer_val(sp, "dash_floor_per_layer", li)
|
| 322 |
+
if scale is None:
|
| 323 |
+
scale = float(sp.get("dash_scaling_factor", router.scaling_factor))
|
| 324 |
+
if floor is None:
|
| 325 |
+
floor = float(sp.get("dash_prior_floor", router._prior_floor))
|
| 326 |
+
router._infer_params = dict(
|
| 327 |
+
idx_scale=float(scale) / math.sqrt(router.index_dim), floor=float(floor),
|
| 328 |
+
sigma=float(sp.get("dash_sigma", router.sigma)), K=int(budget), topm=int(topm))
|
| 329 |
+
state["stage0"].append((attn, router.stage0))
|
| 330 |
+
router.stage0 = stage0
|
| 331 |
+
# vLLM page_state.py isolates PREFILL queries too (image query -> its own page, prompt
|
| 332 |
+
# marker -> raw page, trailing instruction -> unconstrained) == training PAGE_SCOPE_PREFILL=1.
|
| 333 |
+
_base._PAGE_SCOPE_PREFILL = bool(page_scope_prefill)
|
| 334 |
+
_INFER["on"] = True
|
| 335 |
+
return state
|
| 336 |
+
|
| 337 |
+
|
| 338 |
+
@torch.no_grad()
|
| 339 |
+
def _infer_route_select(block_scores, block_cand, p):
|
| 340 |
+
"""vLLM DashRouterMath.route_and_select (budget_topk=True) with training's candidacy.
|
| 341 |
+
|
| 342 |
+
block_scores [B,Lq,nblk] fp32 (_SCORE_NEG = not a remote block); block_cand
|
| 343 |
+
[B,Lq,nblk] bool page-aware candidate domain or None.
|
| 344 |
+
Returns (topi [B,Lq,K] int64, bias [B,Lq,K] fp32, valid [B,Lq,K] bool).
|
| 345 |
+
"""
|
| 346 |
+
remote = block_scores > (_SCORE_NEG * 0.5)
|
| 347 |
+
cand = remote if block_cand is None else (remote & block_cand)
|
| 348 |
+
cand = torch.where(~cand.any(dim=-1, keepdim=True), remote, cand)
|
| 349 |
+
x = torch.where(cand, block_scores * p["idx_scale"], block_scores.new_full((), _ENTMAX_NEG))
|
| 350 |
+
x = torch.nan_to_num(x, nan=_ENTMAX_NEG, posinf=_ENTMAX_NEG, neginf=_ENTMAX_NEG)
|
| 351 |
+
nblk = x.shape[-1]
|
| 352 |
+
M = max(1, min(int(p["topm"]), nblk)) if int(p["topm"]) > 0 else nblk
|
| 353 |
+
xv, xi = x.topk(M, dim=-1) # descending
|
| 354 |
+
prior = _entmax15(xv, dim=-1) # tau* over the top-M window
|
| 355 |
+
is_cand = cand.gather(-1, xi)
|
| 356 |
+
K = max(1, min(int(p["K"]), M))
|
| 357 |
+
xi, prior, valid = xi[..., :K], prior[..., :K], is_cand[..., :K]
|
| 358 |
+
logp = torch.log(prior.clamp_min(p["floor"]))
|
| 359 |
+
cnt = valid.sum(dim=-1, keepdim=True).clamp(min=1)
|
| 360 |
+
mu = torch.where(valid, logp, torch.zeros_like(logp)).sum(dim=-1, keepdim=True) / cnt
|
| 361 |
+
bias = torch.where(valid, (logp - mu) * (1.0 / p["sigma"]), torch.zeros_like(logp))
|
| 362 |
+
return xi, bias.float(), valid
|
| 363 |
+
|
| 364 |
+
|
| 365 |
+
def _router_qk(router, hidden_states, pe):
|
| 366 |
+
"""Router index q/k exactly as DashAttention._block_scores_headcls builds them."""
|
| 367 |
+
B, S, _ = hidden_states.shape
|
| 368 |
+
d = router.index_dim
|
| 369 |
+
hs = torch.nan_to_num(hidden_states, nan=0.0, posinf=0.0, neginf=0.0)
|
| 370 |
+
idx_q = router.q_norm(router.q_proj(hs).view(B, S, 1, d)).transpose(1, 2)
|
| 371 |
+
idx_k = router.k_norm(router.k_proj(hs).view(B, S, 1, d)).transpose(1, 2)
|
| 372 |
+
if pe is not None:
|
| 373 |
+
cos, sin = pe
|
| 374 |
+
idx_q, idx_k = _llama_apply_rotary_pos_emb(idx_q, idx_k, cos[..., :d], sin[..., :d])
|
| 375 |
+
idx_q = torch.nan_to_num(idx_q[:, 0].float(), nan=0.0, posinf=0.0, neginf=0.0)
|
| 376 |
+
idx_k = torch.nan_to_num(idx_k[:, 0].float(), nan=0.0, posinf=0.0, neginf=0.0)
|
| 377 |
+
return idx_q, idx_k
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
@torch.no_grad()
|
| 381 |
+
def _block_scores_infer(router, idx_q, idx_k, q_pos, nblk):
|
| 382 |
+
"""[B,Lq,nblk] fp32 remote block scores for queries ``idx_q`` at positions ``q_pos``
|
| 383 |
+
against ALL keys ``idx_k`` [B,S,d] (prefill: S == Lq; decode: the cached keys).
|
| 384 |
+
Same math as DashAttention._block_scores_headcls / _block_scores."""
|
| 385 |
+
B, S, d = idx_k.shape
|
| 386 |
+
bs = router.block_size
|
| 387 |
+
inv_sqrt_d = 1.0 / math.sqrt(d)
|
| 388 |
+
Sp = nblk * bs
|
| 389 |
+
blk_ids = torch.arange(nblk, device=idx_k.device)
|
| 390 |
+
out = []
|
| 391 |
+
if router.stage0 == "head_cls":
|
| 392 |
+
if Sp > S:
|
| 393 |
+
kk = F.pad(idx_k, (0, 0, 0, Sp - S))
|
| 394 |
+
real = F.pad(idx_k.new_ones(B, S, dtype=torch.bool), (0, Sp - S), value=False)
|
| 395 |
+
else:
|
| 396 |
+
kk = idx_k[:, :Sp]
|
| 397 |
+
real = idx_k.new_ones(B, Sp, dtype=torch.bool)
|
| 398 |
+
kk = kk.view(B, nblk, bs, d)
|
| 399 |
+
vmask = real.view(B, nblk, bs)
|
| 400 |
+
hc = router.head_cls.to(kk.dtype).view(1, 1, 1, d)
|
| 401 |
+
aw = ((kk * hc).sum(-1) * inv_sqrt_d).clamp(-_ACT_CLAMP, _ACT_CLAMP)
|
| 402 |
+
neg = torch.finfo(aw.dtype).min
|
| 403 |
+
row_max = torch.where(vmask, aw, aw.new_full((), neg)).amax(dim=-1, keepdim=True)
|
| 404 |
+
row_max = torch.where(vmask.any(dim=-1, keepdim=True), row_max, torch.zeros_like(row_max))
|
| 405 |
+
e = torch.exp(aw - row_max) * vmask.to(aw.dtype)
|
| 406 |
+
w = e / e.sum(dim=-1, keepdim=True).clamp_min(1e-6)
|
| 407 |
+
k_bar = torch.nan_to_num((w.unsqueeze(-1) * kk).sum(dim=2), nan=0.0, posinf=0.0, neginf=0.0)
|
| 408 |
+
chunk_valid = vmask.any(dim=-1) # [B,nblk]
|
| 409 |
+
for c0 in range(0, idx_q.shape[1], router.q_chunk):
|
| 410 |
+
c1 = min(c0 + router.q_chunk, idx_q.shape[1])
|
| 411 |
+
z = torch.matmul(idx_q[:, c0:c1], k_bar.transpose(-1, -2)) * inv_sqrt_d
|
| 412 |
+
z = z.clamp(-_ACT_CLAMP, _ACT_CLAMP)
|
| 413 |
+
qblk = q_pos[:, c0:c1] // bs
|
| 414 |
+
keep = (blk_ids[None, None, :] <= (qblk[:, :, None] - router.local_blocks)) \
|
| 415 |
+
& chunk_valid[:, None, :]
|
| 416 |
+
out.append(z.masked_fill(~keep, _SCORE_NEG))
|
| 417 |
+
return torch.cat(out, dim=1)
|
| 418 |
+
# stage0 == "indexer": block-max of token-level scores, remote-only (causal minus local)
|
| 419 |
+
kpos = torch.arange(S, device=idx_k.device)
|
| 420 |
+
for c0 in range(0, idx_q.shape[1], router.q_chunk):
|
| 421 |
+
c1 = min(c0 + router.q_chunk, idx_q.shape[1])
|
| 422 |
+
sc = torch.matmul(idx_q[:, c0:c1], idx_k.transpose(-1, -2)).clamp(-_ACT_CLAMP, _ACT_CLAMP)
|
| 423 |
+
pos = q_pos[:, c0:c1]
|
| 424 |
+
future = kpos[None, None, :] > pos[:, :, None]
|
| 425 |
+
local = (kpos[None, None, :] > (pos[:, :, None] - router.local_window)) & ~future
|
| 426 |
+
sc = sc.masked_fill(future | local, _SCORE_NEG)
|
| 427 |
+
if Sp > S:
|
| 428 |
+
sc = F.pad(sc, (0, Sp - S), value=_SCORE_NEG)
|
| 429 |
+
out.append(sc.view(B, c1 - c0, nblk, bs).amax(dim=-1))
|
| 430 |
+
return torch.cat(out, dim=1)
|
| 431 |
+
|
| 432 |
+
|
| 433 |
+
def _infer_prefill_attend(self, router, hidden_states, q, k, v, idx_pe, pos_ids,
|
| 434 |
+
block_cand, attn_mask, past_kv):
|
| 435 |
+
"""Prefill (q_len > 1) under _INFER: vLLM-parity routing, training attend kernels."""
|
| 436 |
+
B, H, S, D = q.shape
|
| 437 |
+
nblk = (S + router.block_size - 1) // router.block_size
|
| 438 |
+
p = router._infer_params
|
| 439 |
+
with torch.autocast(device_type=q.device.type, enabled=False):
|
| 440 |
+
idx_q, idx_k = _router_qk(router, hidden_states, idx_pe)
|
| 441 |
+
if past_kv is not None:
|
| 442 |
+
if not hasattr(past_kv, "_dash_idx_k"):
|
| 443 |
+
past_kv._dash_idx_k = {}
|
| 444 |
+
past_kv._dash_idx_k[self.layer_idx] = idx_k
|
| 445 |
+
sc = _block_scores_infer(router, idx_q, idx_k, pos_ids, nblk)
|
| 446 |
+
topi, bias, valid = _infer_route_select(sc, block_cand, p)
|
| 447 |
+
if router.use_triton and q.is_cuda and attn_mask is None:
|
| 448 |
+
out = router._forward_triton(q, k, v, topi, bias, valid, S, nblk, pos_ids)
|
| 449 |
+
if out is not None:
|
| 450 |
+
return out
|
| 451 |
+
return router._forward_dense(q, k, v, topi, bias, valid, S, nblk, attn_mask)
|
| 452 |
+
|
| 453 |
+
|
| 454 |
+
@torch.no_grad()
|
| 455 |
+
def _decode_attend(q, k_all, v_all, topi, bias, valid, q_pos, block_size, local_window, scale):
|
| 456 |
+
"""Bias-aware block-sparse attention for decode queries (fp32 math).
|
| 457 |
+
|
| 458 |
+
Selected remote blocks add their ``bias``; the token-level local window
|
| 459 |
+
(q_pos - local_window, q_pos] is attended without bias and wins on overlap --
|
| 460 |
+
same read set / weights as vLLM dash_sparse_attn decode and training _forward_dense.
|
| 461 |
+
"""
|
| 462 |
+
B, H, Lq, D = q.shape
|
| 463 |
+
S = k_all.shape[2]
|
| 464 |
+
nblk = (S + block_size - 1) // block_size
|
| 465 |
+
kpos = torch.arange(S, device=q.device)
|
| 466 |
+
src = torch.where(valid, bias, bias.new_full((), _MASK_NEG))
|
| 467 |
+
blk = torch.full((B, Lq, nblk), _MASK_NEG, device=q.device, dtype=torch.float32)
|
| 468 |
+
blk = blk.scatter(-1, topi.long(), src.float())
|
| 469 |
+
base = blk.index_select(-1, kpos // block_size) # [B,Lq,S]
|
| 470 |
+
qp = q_pos[:, :, None]
|
| 471 |
+
causal = kpos[None, None, :] <= qp
|
| 472 |
+
local = causal & (kpos[None, None, :] > (qp - local_window))
|
| 473 |
+
keep = causal & (local | (base > _MASK_NEG * 0.5))
|
| 474 |
+
m = torch.where(local, torch.zeros_like(base), base).masked_fill(~keep, float("-inf"))
|
| 475 |
+
kk = k_all.float()
|
| 476 |
+
vv = v_all.float()
|
| 477 |
+
if kk.shape[1] != H:
|
| 478 |
+
kk = _llama_repeat_kv(kk, H // kk.shape[1])
|
| 479 |
+
vv = _llama_repeat_kv(vv, H // vv.shape[1])
|
| 480 |
+
s = torch.matmul(q.float(), kk.transpose(-1, -2)) * scale + m[:, None]
|
| 481 |
+
return torch.matmul(torch.softmax(s, dim=-1), vv) # [B,H,Lq,D]
|
| 482 |
+
|
| 483 |
+
|
| 484 |
+
def _dash_decode_forward(self, hidden_states, position_ids, past_kv):
|
| 485 |
+
"""Decode step (q_len == 1) of a DASH layer under _INFER (vLLM DASH_DECODE_DENSE=0)."""
|
| 486 |
+
idx = self.indexer
|
| 487 |
+
router = self.dash_router
|
| 488 |
+
nh = self.config.num_attention_heads
|
| 489 |
+
nkv = self.config.num_key_value_heads
|
| 490 |
+
hd = self.head_dim
|
| 491 |
+
bsz, q_len, _ = hidden_states.size()
|
| 492 |
+
if position_ids is None:
|
| 493 |
+
past = past_kv.get_seq_length(self.layer_idx)
|
| 494 |
+
position_ids = torch.arange(past, past + q_len, device=hidden_states.device)[None].expand(bsz, -1)
|
| 495 |
+
|
| 496 |
+
q = self.q_proj(hidden_states).view(bsz, q_len, nh, hd).transpose(1, 2)
|
| 497 |
+
k = self.k_proj(hidden_states).view(bsz, q_len, nkv, hd).transpose(1, 2)
|
| 498 |
+
v = self.v_proj(hidden_states).view(bsz, q_len, nkv, hd).transpose(1, 2)
|
| 499 |
+
|
| 500 |
+
# (tok_pos, page_pos) from the incremental decode state machine (advances it once).
|
| 501 |
+
coords = None
|
| 502 |
+
if self.main_rope_mode == "page2d" and self.rotary_emb_2d_main is not None:
|
| 503 |
+
coords = self._main_rope_coords(q_len, past_kv)
|
| 504 |
+
if coords is not None:
|
| 505 |
+
cos, sin = self.rotary_emb_2d_main(v, coords[0], coords[1])
|
| 506 |
+
else:
|
| 507 |
+
cos, sin = self.rotary_emb(v, position_ids)
|
| 508 |
+
q, k = _llama_apply_rotary_pos_emb(q, k, cos, sin)
|
| 509 |
+
k_all, v_all = past_kv.update(k, v, self.layer_idx)
|
| 510 |
+
idx_pe = _dash_indexer_rope(self, idx, router, v, position_ids,
|
| 511 |
+
coords if idx.rope_mode == "page2d" else None, (cos, sin))
|
| 512 |
+
|
| 513 |
+
p = router._infer_params
|
| 514 |
+
with torch.autocast(device_type=hidden_states.device.type, enabled=False):
|
| 515 |
+
idx_q, idx_k_new = _router_qk(router, hidden_states, idx_pe)
|
| 516 |
+
idx_k = torch.cat([past_kv._dash_idx_k[self.layer_idx], idx_k_new], dim=1)
|
| 517 |
+
past_kv._dash_idx_k[self.layer_idx] = idx_k
|
| 518 |
+
S = k_all.shape[2]
|
| 519 |
+
nblk = (S + router.block_size - 1) // router.block_size
|
| 520 |
+
sc = _block_scores_infer(router, idx_q, idx_k, position_ids, nblk) # [B,Lq,nblk]
|
| 521 |
+
allow = getattr(self, "_dash_dec_allow", None) # [B,nblk] or None
|
| 522 |
+
cand = None
|
| 523 |
+
if allow is not None and router.image_only:
|
| 524 |
+
if allow.shape[-1] < nblk:
|
| 525 |
+
allow = F.pad(allow, (0, nblk - allow.shape[-1]), value=False)
|
| 526 |
+
cand = allow[:, None, :nblk].expand(bsz, q_len, nblk)
|
| 527 |
+
topi, bias, valid = _infer_route_select(sc, cand, p)
|
| 528 |
+
out = _decode_attend(q, k_all, v_all, topi, bias, valid, position_ids,
|
| 529 |
+
router.block_size, router.local_window, router._scale)
|
| 530 |
+
out = out.to(hidden_states.dtype).transpose(1, 2).contiguous().reshape(bsz, q_len, -1)
|
| 531 |
+
return self.o_proj(out), None, past_kv, None
|
modeling_unlimitedocr.py
ADDED
|
@@ -0,0 +1,1228 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .modeling_deepseekv2 import (
|
| 2 |
+
DeepseekV2Model, DeepseekV2ForCausalLM, _fused_chunked_lm_ce,
|
| 3 |
+
_td, _td_now, _td_add, _td_report,
|
| 4 |
+
)
|
| 5 |
+
from .modeling_deepseekv2 import _INFER as _DASH_INFER, configure_dash_inference
|
| 6 |
+
from .configuration_deepseek_v2 import DeepseekV2Config
|
| 7 |
+
# transformers' dynamic-module loader only copies DIRECT relative imports
|
| 8 |
+
# of the entry module to the HF cache for local model dirs (no recursion). Force
|
| 9 |
+
# sparse_gather_triton (used lazily inside modeling_deepseekv2) to be copied by
|
| 10 |
+
# importing it here at the entry point. The regex in transformers only matches
|
| 11 |
+
# ``from .xxx import yyy`` (not ``from . import xxx``), so keep this exact form.
|
| 12 |
+
try:
|
| 13 |
+
from .sparse_gather_triton import SPARSE_BLOCK_SIZE_SENTINEL as _sgt_sentinel # noqa: F401
|
| 14 |
+
except Exception:
|
| 15 |
+
pass
|
| 16 |
+
# same reason — packing_patch is used inside modeling_deepseekv2
|
| 17 |
+
# (2nd-level import), so force it to be copied by importing it here at the entry.
|
| 18 |
+
try:
|
| 19 |
+
from .packing_patch import is_packed as _pk_copy_hint # noqa: F401
|
| 20 |
+
except Exception:
|
| 21 |
+
pass
|
| 22 |
+
# Same reason for the dash overlay: _sparse_base_impl (the base implementation),
|
| 23 |
+
# dash_attention (entmax router) and sparse_gather_e2e_triton (fused kernel with the
|
| 24 |
+
# per-block bias) are 2nd-level relative imports as well, so they are only copied
|
| 25 |
+
# into the HF cache when the entry module imports them directly.
|
| 26 |
+
try:
|
| 27 |
+
from ._sparse_base_impl import DeepseekV2Model as _dash_base_copy_hint # noqa: F401
|
| 28 |
+
except Exception:
|
| 29 |
+
pass
|
| 30 |
+
try:
|
| 31 |
+
from .dash_attention import DashAttention as _dash_router_copy_hint # noqa: F401
|
| 32 |
+
except Exception:
|
| 33 |
+
pass
|
| 34 |
+
try:
|
| 35 |
+
from .sparse_gather_e2e_triton import triton_gather_attention_e2e as _dash_e2e_copy_hint # noqa: F401
|
| 36 |
+
except Exception:
|
| 37 |
+
pass
|
| 38 |
+
from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
|
| 39 |
+
from typing import List, Optional, Tuple, Union
|
| 40 |
+
from dataclasses import dataclass
|
| 41 |
+
|
| 42 |
+
from transformers.cache_utils import Cache
|
| 43 |
+
import requests
|
| 44 |
+
from PIL import Image, ImageOps, ImageDraw, ImageFont
|
| 45 |
+
from io import BytesIO
|
| 46 |
+
import torch
|
| 47 |
+
import torch.nn as nn
|
| 48 |
+
import torch.nn.functional as F
|
| 49 |
+
from torch.nn import CrossEntropyLoss
|
| 50 |
+
from torchvision import transforms
|
| 51 |
+
from torchvision.transforms.functional import InterpolationMode
|
| 52 |
+
import os
|
| 53 |
+
from .deepencoder import build_sam_vit_b, build_clip_l, MlpProjector
|
| 54 |
+
from addict import Dict
|
| 55 |
+
from transformers import TextStreamer
|
| 56 |
+
from .conversation import get_conv_template
|
| 57 |
+
from abc import ABC
|
| 58 |
+
import math
|
| 59 |
+
import re
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
@dataclass
|
| 63 |
+
class CausalLMOutputWithPastAndKL(CausalLMOutputWithPast):
|
| 64 |
+
"""ForCausalLM output with an extra `indexer_kl_loss` field so the sparse
|
| 65 |
+
indexer's KL alignment loss survives DDP's pytree flatten/unflatten (which strips
|
| 66 |
+
any dynamic attributes attached to a plain `CausalLMOutputWithPast`).
|
| 67 |
+
Read by `swift/trainers/seq2seq_trainer.py:compute_loss`.
|
| 68 |
+
"""
|
| 69 |
+
indexer_kl_loss: Optional[torch.FloatTensor] = None
|
| 70 |
+
# raw fused-CE token-accuracy counts (correct / valid). Kept as separate
|
| 71 |
+
# counts (not a pre-averaged ratio) so the trainer can token-weight the accuracy
|
| 72 |
+
# exactly like swift's native compute_acc path.
|
| 73 |
+
token_acc_correct: Optional[torch.FloatTensor] = None
|
| 74 |
+
token_acc_valid: Optional[torch.FloatTensor] = None
|
| 75 |
+
|
| 76 |
+
from tqdm import tqdm
|
| 77 |
+
import numpy as np
|
| 78 |
+
import time
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def load_image(image_path):
|
| 82 |
+
|
| 83 |
+
try:
|
| 84 |
+
image = Image.open(image_path)
|
| 85 |
+
|
| 86 |
+
corrected_image = ImageOps.exif_transpose(image)
|
| 87 |
+
|
| 88 |
+
return corrected_image
|
| 89 |
+
|
| 90 |
+
except Exception as e:
|
| 91 |
+
print(f"error: {e}")
|
| 92 |
+
try:
|
| 93 |
+
return Image.open(image_path)
|
| 94 |
+
except:
|
| 95 |
+
return None
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def re_match(text):
|
| 99 |
+
ref_pattern = r'(<\|ref\|>(.*?)<\|/ref\|><\|det\|>(.*?)<\|/det\|>)'
|
| 100 |
+
matches = re.findall(ref_pattern, text, re.DOTALL)
|
| 101 |
+
|
| 102 |
+
det_pattern = r'(<\|det\|>\s*([A-Za-z_][\w-]*)\s*(\[[^\]]+\])\s*<\|/det\|>)'
|
| 103 |
+
for full_match, label, box in re.findall(det_pattern, text, re.DOTALL):
|
| 104 |
+
matches.append((full_match, label, box))
|
| 105 |
+
|
| 106 |
+
mathes_image = []
|
| 107 |
+
mathes_other = []
|
| 108 |
+
for a_match in matches:
|
| 109 |
+
if a_match[1].strip() == 'image' or '<|ref|>image<|/ref|>' in a_match[0]:
|
| 110 |
+
mathes_image.append(a_match[0])
|
| 111 |
+
else:
|
| 112 |
+
mathes_other.append(a_match[0])
|
| 113 |
+
return matches, mathes_image, mathes_other
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def extract_coordinates_and_label(ref_text, image_width, image_height):
|
| 117 |
+
|
| 118 |
+
try:
|
| 119 |
+
label_type = ref_text[1]
|
| 120 |
+
cor_list = eval(ref_text[2])
|
| 121 |
+
if cor_list and isinstance(cor_list[0], (int, float)):
|
| 122 |
+
cor_list = [cor_list]
|
| 123 |
+
except Exception as e:
|
| 124 |
+
print(e)
|
| 125 |
+
return None
|
| 126 |
+
|
| 127 |
+
return (label_type, cor_list)
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def draw_bounding_boxes(image, refs, ouput_path, image_prefix=''):
|
| 131 |
+
|
| 132 |
+
image_width, image_height = image.size
|
| 133 |
+
|
| 134 |
+
img_draw = image.copy()
|
| 135 |
+
draw = ImageDraw.Draw(img_draw)
|
| 136 |
+
|
| 137 |
+
overlay = Image.new('RGBA', img_draw.size, (0, 0, 0, 0))
|
| 138 |
+
draw2 = ImageDraw.Draw(overlay)
|
| 139 |
+
|
| 140 |
+
# try:
|
| 141 |
+
# except IOError:
|
| 142 |
+
# try:
|
| 143 |
+
# font = ImageFont.truetype("DejaVuSans.ttf", 20)
|
| 144 |
+
# except IOError:
|
| 145 |
+
font = ImageFont.load_default()
|
| 146 |
+
|
| 147 |
+
img_idx = 0
|
| 148 |
+
|
| 149 |
+
for i, ref in enumerate(refs):
|
| 150 |
+
try:
|
| 151 |
+
result = extract_coordinates_and_label(ref, image_width, image_height)
|
| 152 |
+
if result:
|
| 153 |
+
label_type, points_list = result
|
| 154 |
+
|
| 155 |
+
color = (np.random.randint(0, 200), np.random.randint(0, 200), np.random.randint(0, 255))
|
| 156 |
+
|
| 157 |
+
color_a = color + (20, )
|
| 158 |
+
for points in points_list:
|
| 159 |
+
x1, y1, x2, y2 = points
|
| 160 |
+
|
| 161 |
+
x1 = int(x1 / 999 * image_width)
|
| 162 |
+
y1 = int(y1 / 999 * image_height)
|
| 163 |
+
|
| 164 |
+
x2 = int(x2 / 999 * image_width)
|
| 165 |
+
y2 = int(y2 / 999 * image_height)
|
| 166 |
+
|
| 167 |
+
if label_type == 'image':
|
| 168 |
+
try:
|
| 169 |
+
cropped = image.crop((x1, y1, x2, y2))
|
| 170 |
+
cropped.save(f"{ouput_path}/images/{image_prefix}{img_idx}.jpg")
|
| 171 |
+
except Exception as e:
|
| 172 |
+
print(e)
|
| 173 |
+
pass
|
| 174 |
+
img_idx += 1
|
| 175 |
+
|
| 176 |
+
try:
|
| 177 |
+
if label_type == 'title':
|
| 178 |
+
draw.rectangle([x1, y1, x2, y2], outline=color, width=4)
|
| 179 |
+
draw2.rectangle([x1, y1, x2, y2], fill=color_a, outline=(0, 0, 0, 0), width=1)
|
| 180 |
+
else:
|
| 181 |
+
draw.rectangle([x1, y1, x2, y2], outline=color, width=2)
|
| 182 |
+
draw2.rectangle([x1, y1, x2, y2], fill=color_a, outline=(0, 0, 0, 0), width=1)
|
| 183 |
+
text_x = x1
|
| 184 |
+
text_y = max(0, y1 - 15)
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
text_bbox = draw.textbbox((0, 0), label_type, font=font)
|
| 188 |
+
text_width = text_bbox[2] - text_bbox[0]
|
| 189 |
+
text_height = text_bbox[3] - text_bbox[1]
|
| 190 |
+
draw.rectangle([text_x, text_y, text_x + text_width, text_y + text_height],
|
| 191 |
+
fill=(255, 255, 255, 30))
|
| 192 |
+
|
| 193 |
+
draw.text((text_x, text_y), label_type, font=font, fill=color)
|
| 194 |
+
except:
|
| 195 |
+
pass
|
| 196 |
+
except:
|
| 197 |
+
continue
|
| 198 |
+
img_draw.paste(overlay, (0, 0), overlay)
|
| 199 |
+
return img_draw
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def process_image_with_refs(image, ref_texts, output_path, image_prefix=''):
|
| 203 |
+
|
| 204 |
+
result_image = draw_bounding_boxes(image, ref_texts, output_path, image_prefix=image_prefix)
|
| 205 |
+
|
| 206 |
+
return result_image
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
|
| 213 |
+
best_ratio_diff = float('inf')
|
| 214 |
+
best_ratio = (1, 1)
|
| 215 |
+
area = width * height
|
| 216 |
+
for ratio in target_ratios:
|
| 217 |
+
target_aspect_ratio = ratio[0] / ratio[1]
|
| 218 |
+
ratio_diff = abs(aspect_ratio - target_aspect_ratio)
|
| 219 |
+
if ratio_diff < best_ratio_diff:
|
| 220 |
+
best_ratio_diff = ratio_diff
|
| 221 |
+
best_ratio = ratio
|
| 222 |
+
elif ratio_diff == best_ratio_diff:
|
| 223 |
+
if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
|
| 224 |
+
best_ratio = ratio
|
| 225 |
+
# print(f'width: {width}, height: {height}, best_ratio: {best_ratio}')
|
| 226 |
+
return best_ratio
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def dynamic_preprocess(image, min_num=2, max_num=16, image_size=640, use_thumbnail=False):
|
| 230 |
+
orig_width, orig_height = image.size
|
| 231 |
+
aspect_ratio = orig_width / orig_height
|
| 232 |
+
|
| 233 |
+
# calculate the existing image aspect ratio
|
| 234 |
+
target_ratios = set(
|
| 235 |
+
(i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
|
| 236 |
+
i * j <= max_num and i * j >= min_num)
|
| 237 |
+
# print(target_ratios)
|
| 238 |
+
target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
|
| 239 |
+
|
| 240 |
+
# find the closest aspect ratio to the target
|
| 241 |
+
target_aspect_ratio = find_closest_aspect_ratio(
|
| 242 |
+
aspect_ratio, target_ratios, orig_width, orig_height, image_size)
|
| 243 |
+
|
| 244 |
+
# print(target_aspect_ratio)
|
| 245 |
+
# calculate the target width and height
|
| 246 |
+
target_width = image_size * target_aspect_ratio[0]
|
| 247 |
+
target_height = image_size * target_aspect_ratio[1]
|
| 248 |
+
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
|
| 249 |
+
|
| 250 |
+
# resize the image
|
| 251 |
+
resized_img = image.resize((target_width, target_height))
|
| 252 |
+
processed_images = []
|
| 253 |
+
for i in range(blocks):
|
| 254 |
+
box = (
|
| 255 |
+
(i % (target_width // image_size)) * image_size,
|
| 256 |
+
(i // (target_width // image_size)) * image_size,
|
| 257 |
+
((i % (target_width // image_size)) + 1) * image_size,
|
| 258 |
+
((i // (target_width // image_size)) + 1) * image_size
|
| 259 |
+
)
|
| 260 |
+
# split the image
|
| 261 |
+
split_img = resized_img.crop(box)
|
| 262 |
+
processed_images.append(split_img)
|
| 263 |
+
assert len(processed_images) == blocks
|
| 264 |
+
if use_thumbnail and len(processed_images) != 1:
|
| 265 |
+
thumbnail_img = image.resize((image_size, image_size))
|
| 266 |
+
processed_images.append(thumbnail_img)
|
| 267 |
+
return processed_images, target_aspect_ratio
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
def normalize_transform(mean, std):
|
| 272 |
+
if mean is None and std is None:
|
| 273 |
+
transform = None
|
| 274 |
+
elif mean is None and std is not None:
|
| 275 |
+
mean = [0.] * len(std)
|
| 276 |
+
transform = transforms.Normalize(mean=mean, std=std)
|
| 277 |
+
elif mean is not None and std is None:
|
| 278 |
+
std = [1.] * len(mean)
|
| 279 |
+
transform = transforms.Normalize(mean=mean, std=std)
|
| 280 |
+
else:
|
| 281 |
+
transform = transforms.Normalize(mean=mean, std=std)
|
| 282 |
+
|
| 283 |
+
return transform
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def format_messages(
|
| 288 |
+
conversations: List[Dict[str, str]],
|
| 289 |
+
sft_format: str = "deepseek",
|
| 290 |
+
system_prompt: str = "",
|
| 291 |
+
):
|
| 292 |
+
"""
|
| 293 |
+
Applies the SFT template to conversation.
|
| 294 |
+
|
| 295 |
+
Args:
|
| 296 |
+
conversations (List[Dict]): A List of messages.
|
| 297 |
+
sft_format (str, optional): The format of the SFT template to use. Defaults to "deepseek".
|
| 298 |
+
system_prompt (str, optional): The system prompt to use in the SFT template. Defaults to "".
|
| 299 |
+
|
| 300 |
+
Returns:
|
| 301 |
+
sft_prompt (str): The formatted text.
|
| 302 |
+
"""
|
| 303 |
+
|
| 304 |
+
conv = get_conv_template(sft_format)
|
| 305 |
+
conv.set_system_message(system_prompt)
|
| 306 |
+
for message in conversations:
|
| 307 |
+
conv.append_message(message["role"], message["content"].strip())
|
| 308 |
+
sft_prompt = conv.get_prompt().strip()
|
| 309 |
+
|
| 310 |
+
return sft_prompt
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
def text_encode(tokenizer, text: str, bos: bool = True, eos: bool = False):
|
| 314 |
+
t = tokenizer.encode(text, add_special_tokens=False)
|
| 315 |
+
bos_id = 0
|
| 316 |
+
eos_id = 1
|
| 317 |
+
if bos:
|
| 318 |
+
t = [bos_id] + t
|
| 319 |
+
if eos:
|
| 320 |
+
t = t + [eos_id]
|
| 321 |
+
|
| 322 |
+
return t
|
| 323 |
+
|
| 324 |
+
def load_pil_images(conversations: List[Dict[str, str]]) -> List[Image.Image]:
|
| 325 |
+
"""
|
| 326 |
+
|
| 327 |
+
Args:
|
| 328 |
+
conversations (List[Dict[str, str]]): the conversations with a list of messages. An example is :
|
| 329 |
+
[
|
| 330 |
+
{
|
| 331 |
+
"role": "User",
|
| 332 |
+
"content": "<image_placeholder>\nExtract all information from this image and convert them into markdown format.",
|
| 333 |
+
"images": ["./examples/table_datasets.png"]
|
| 334 |
+
},
|
| 335 |
+
{"role": "Assistant", "content": ""},
|
| 336 |
+
]
|
| 337 |
+
|
| 338 |
+
Returns:
|
| 339 |
+
pil_images (List[PIL.Image.Image]): the list of PIL images.
|
| 340 |
+
|
| 341 |
+
"""
|
| 342 |
+
|
| 343 |
+
pil_images = []
|
| 344 |
+
|
| 345 |
+
for message in conversations:
|
| 346 |
+
if "images" not in message:
|
| 347 |
+
continue
|
| 348 |
+
|
| 349 |
+
for image_path in message["images"]:
|
| 350 |
+
# print('----------------')
|
| 351 |
+
# print(image_path)
|
| 352 |
+
# print('----------------')
|
| 353 |
+
# exit()
|
| 354 |
+
|
| 355 |
+
# pil_img = Image.open(image_path)
|
| 356 |
+
pil_img = load_image(image_path)
|
| 357 |
+
pil_img = pil_img.convert("RGB")
|
| 358 |
+
pil_images.append(pil_img)
|
| 359 |
+
|
| 360 |
+
return pil_images
|
| 361 |
+
|
| 362 |
+
|
| 363 |
+
class BaseTransform(ABC):
|
| 364 |
+
|
| 365 |
+
def set_rng(self, *args, **kwargs):
|
| 366 |
+
pass
|
| 367 |
+
|
| 368 |
+
def __call__(self, *args, **kwargs) -> torch.Tensor:
|
| 369 |
+
pass
|
| 370 |
+
|
| 371 |
+
@property
|
| 372 |
+
def default_shape(self):
|
| 373 |
+
raise NotImplementedError
|
| 374 |
+
|
| 375 |
+
|
| 376 |
+
class BasicImageTransform(BaseTransform):
|
| 377 |
+
def __init__(
|
| 378 |
+
self,
|
| 379 |
+
mean: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5),
|
| 380 |
+
std: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5),
|
| 381 |
+
normalize: bool = True
|
| 382 |
+
):
|
| 383 |
+
self.mean = mean
|
| 384 |
+
self.std = std
|
| 385 |
+
|
| 386 |
+
transform_pipelines = [
|
| 387 |
+
transforms.ToTensor()
|
| 388 |
+
]
|
| 389 |
+
|
| 390 |
+
normalize = normalize_transform(mean, std) if normalize else nn.Identity()
|
| 391 |
+
if normalize is not None:
|
| 392 |
+
transform_pipelines.append(normalize)
|
| 393 |
+
|
| 394 |
+
self.transform = transforms.Compose(transform_pipelines)
|
| 395 |
+
|
| 396 |
+
def __call__(self, x):
|
| 397 |
+
x = self.transform(x)
|
| 398 |
+
return x
|
| 399 |
+
|
| 400 |
+
class NoEOSTextStreamer(TextStreamer):
|
| 401 |
+
def on_finalized_text(self, text: str, stream_end: bool = False):
|
| 402 |
+
|
| 403 |
+
eos_text = self.tokenizer.decode([self.tokenizer.eos_token_id], skip_special_tokens=False)
|
| 404 |
+
text = text.replace(eos_text, "\n")
|
| 405 |
+
print(text, flush=True, end="")
|
| 406 |
+
|
| 407 |
+
|
| 408 |
+
class SlidingWindowNoRepeatNgramProcessor:
|
| 409 |
+
"""Block n-gram repetitions within a sliding window.
|
| 410 |
+
Aligned with SGLang DeepseekOCRNoRepeatNGramLogitProcessor."""
|
| 411 |
+
def __init__(self, ngram_size, window, whitelist_token_ids=None):
|
| 412 |
+
self.ngram_size = ngram_size
|
| 413 |
+
self.window = window
|
| 414 |
+
self.whitelist = set(whitelist_token_ids) if whitelist_token_ids else set()
|
| 415 |
+
|
| 416 |
+
def __call__(self, input_ids, scores):
|
| 417 |
+
for batch_idx in range(input_ids.shape[0]):
|
| 418 |
+
sequence = input_ids[batch_idx].tolist()
|
| 419 |
+
if len(sequence) < self.ngram_size:
|
| 420 |
+
continue
|
| 421 |
+
search_start = max(0, len(sequence) - self.window)
|
| 422 |
+
search_end = len(sequence) - self.ngram_size + 1
|
| 423 |
+
if search_end <= search_start:
|
| 424 |
+
continue
|
| 425 |
+
if self.ngram_size > 1:
|
| 426 |
+
current_prefix = tuple(sequence[-(self.ngram_size - 1):])
|
| 427 |
+
else:
|
| 428 |
+
current_prefix = tuple()
|
| 429 |
+
banned = set()
|
| 430 |
+
for idx in range(search_start, search_end):
|
| 431 |
+
ngram = sequence[idx:idx + self.ngram_size]
|
| 432 |
+
if self.ngram_size == 1 or tuple(ngram[:-1]) == current_prefix:
|
| 433 |
+
banned.add(ngram[-1])
|
| 434 |
+
banned.difference_update(self.whitelist)
|
| 435 |
+
for token_id in banned:
|
| 436 |
+
scores[batch_idx, token_id] = float('-inf')
|
| 437 |
+
return scores
|
| 438 |
+
|
| 439 |
+
|
| 440 |
+
class TPSTextStreamer(TextStreamer):
|
| 441 |
+
"""Streamer that prints TPS every `interval` tokens. Set interval=0 to disable."""
|
| 442 |
+
def __init__(self, tokenizer, interval=100, **kwargs):
|
| 443 |
+
super().__init__(tokenizer, **kwargs)
|
| 444 |
+
self.interval = interval
|
| 445 |
+
self.token_count = 0
|
| 446 |
+
self.start_time = None
|
| 447 |
+
self.start_token_count = 0
|
| 448 |
+
self.last_report_count = 0
|
| 449 |
+
self.last_report_time = None
|
| 450 |
+
|
| 451 |
+
def put(self, value):
|
| 452 |
+
import time
|
| 453 |
+
if hasattr(value, 'numel'):
|
| 454 |
+
self.token_count += value.numel()
|
| 455 |
+
else:
|
| 456 |
+
self.token_count += 1
|
| 457 |
+
if self.start_time is None:
|
| 458 |
+
self.start_time = time.time()
|
| 459 |
+
self.last_report_time = self.start_time
|
| 460 |
+
self.last_report_count = self.token_count
|
| 461 |
+
self.start_token_count = self.token_count
|
| 462 |
+
super().put(value)
|
| 463 |
+
return
|
| 464 |
+
if self.interval > 0 and self.token_count - self.last_report_count >= self.interval:
|
| 465 |
+
now = time.time()
|
| 466 |
+
delta_tokens = self.token_count - self.last_report_count
|
| 467 |
+
delta_time = now - self.last_report_time
|
| 468 |
+
recent_tps = delta_tokens / delta_time if delta_time > 0 else 0
|
| 469 |
+
avg_tps = (self.token_count - self.start_token_count) / (now - self.start_time) if (now - self.start_time) > 0 else 0
|
| 470 |
+
print(f"\n[TPS] tokens={self.token_count}, recent={recent_tps:.1f} t/s, avg={avg_tps:.1f} t/s", flush=True)
|
| 471 |
+
self.last_report_count = self.token_count
|
| 472 |
+
self.last_report_time = now
|
| 473 |
+
super().put(value)
|
| 474 |
+
|
| 475 |
+
def on_finalized_text(self, text: str, stream_end: bool = False):
|
| 476 |
+
eos_text = self.tokenizer.decode([self.tokenizer.eos_token_id], skip_special_tokens=False)
|
| 477 |
+
text = text.replace(eos_text, "\n")
|
| 478 |
+
print(text, flush=True, end="")
|
| 479 |
+
|
| 480 |
+
|
| 481 |
+
@torch.no_grad()
|
| 482 |
+
def _uocr_dash_decode_allow(indexer, input_ids, cache):
|
| 483 |
+
"""Page-aware remote candidate blocks of the DECODE query (vLLM page_state.py parity).
|
| 484 |
+
|
| 485 |
+
Prefill (q_len > 1): record, per KV block, the [min, max] image page it holds, the
|
| 486 |
+
prompt page count ``pp`` (<PAGE> markers up to the last image token) and the marker
|
| 487 |
+
count. Returns None (prefill builds its own per-query candidates).
|
| 488 |
+
Decode (q_len == 1): the query token's page is ``marker_cum - 1 - pp`` (response pages
|
| 489 |
+
restart at 0, so text page k <-> image page k); the candidate set is the blocks holding
|
| 490 |
+
an image token of that page, or every block when the page is unknown / has no image
|
| 491 |
+
block (training's per-query fallback). Returns [B, nblk] bool.
|
| 492 |
+
"""
|
| 493 |
+
img_id, pg_id = indexer.image_token_id, indexer.page_token_id
|
| 494 |
+
if img_id is None or pg_id is None:
|
| 495 |
+
return None
|
| 496 |
+
bs = int(indexer.block_size)
|
| 497 |
+
ids = input_ids
|
| 498 |
+
B, L = ids.shape
|
| 499 |
+
if L > 1:
|
| 500 |
+
img = ids == img_id
|
| 501 |
+
cum = torch.cumsum((ids == pg_id).long(), dim=-1)
|
| 502 |
+
run = torch.cumsum(img.long(), dim=-1)
|
| 503 |
+
per_page = max(1, int(indexer.per_page_image_tokens))
|
| 504 |
+
page = torch.where(cum > 0, cum - 1, (run - 1) // per_page)
|
| 505 |
+
page = torch.where(img, page, torch.full_like(page, -1))
|
| 506 |
+
pp = torch.where(img, cum, torch.zeros_like(cum)).cummax(dim=-1).values[:, -1]
|
| 507 |
+
nblk = -(-L // bs)
|
| 508 |
+
pg = F.pad(page, (0, nblk * bs - L), value=-1).view(B, nblk, bs)
|
| 509 |
+
bmax = pg.amax(dim=-1)
|
| 510 |
+
bmin = torch.where(pg >= 0, pg, torch.full_like(pg, 1 << 30)).amin(dim=-1)
|
| 511 |
+
bmin = torch.where(bmax >= 0, bmin, torch.full_like(bmin, -1))
|
| 512 |
+
cache._uocr_pg = {'bmin': bmin, 'bmax': bmax, 'pp': pp, 'mcum': cum[:, -1].clone()}
|
| 513 |
+
return None
|
| 514 |
+
st = getattr(cache, '_uocr_pg', None)
|
| 515 |
+
if st is None:
|
| 516 |
+
return None
|
| 517 |
+
st['mcum'] = st['mcum'] + (ids[:, -1] == pg_id).long()
|
| 518 |
+
qp = st['mcum'] - 1 - st['pp']
|
| 519 |
+
qp = torch.where(qp >= 0, qp, torch.full_like(qp, -1))[:, None]
|
| 520 |
+
total = int(cache.get_seq_length()) + L
|
| 521 |
+
nblk = -(-total // bs)
|
| 522 |
+
bmin, bmax = st['bmin'], st['bmax']
|
| 523 |
+
same = (bmin >= 0) & (bmin <= qp) & (bmax >= qp) & (qp >= 0)
|
| 524 |
+
open_row = (qp[:, 0] < 0) | ~same.any(dim=-1)
|
| 525 |
+
allow = torch.zeros(B, nblk, dtype=torch.bool, device=ids.device)
|
| 526 |
+
n0 = min(nblk, same.shape[-1])
|
| 527 |
+
allow[:, :n0] = same[:, :n0]
|
| 528 |
+
return allow | open_row[:, None]
|
| 529 |
+
|
| 530 |
+
|
| 531 |
+
def _uocr_build_inputs(tokenizer, prompt, images, crop_mode=None, base_size=1024, image_size=640,
|
| 532 |
+
min_crops=2, max_crops=16, patch_size=16, downsample_ratio=4,
|
| 533 |
+
image_token='<image>', image_token_id=128815, bos_id=0):
|
| 534 |
+
"""Tokens + pixel tensors exactly like vLLM's UnlimitedOCRProcessor.tokenize_with_images.
|
| 535 |
+
|
| 536 |
+
crop_mode None == vLLM auto policy: gundam (dynamic tiles) for a single image, base
|
| 537 |
+
(one letterbox-padded global view) for multi-image. No 640x640 pre-resize in base mode
|
| 538 |
+
(vLLM's training-parity fix): the original page is padded straight into base_size.
|
| 539 |
+
"""
|
| 540 |
+
n_img = len(images)
|
| 541 |
+
cropping = (n_img == 1) if crop_mode is None else bool(crop_mode)
|
| 542 |
+
splits = prompt.split(image_token)
|
| 543 |
+
assert len(splits) == n_img + 1, f'prompt has {len(splits) - 1} {image_token} but {n_img} images'
|
| 544 |
+
transform = BasicImageTransform(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5), normalize=True)
|
| 545 |
+
pad_color = tuple(int(x * 255) for x in transform.mean)
|
| 546 |
+
nq = math.ceil((image_size // patch_size) / downsample_ratio)
|
| 547 |
+
nqb = math.ceil((base_size // patch_size) / downsample_ratio)
|
| 548 |
+
ids, seq_mask, globals_, crops, spatial = [], [], [], [], []
|
| 549 |
+
for text_sep, image in zip(splits, images):
|
| 550 |
+
t = tokenizer.encode(text_sep, add_special_tokens=False)
|
| 551 |
+
ids += t
|
| 552 |
+
seq_mask += [False] * len(t)
|
| 553 |
+
tiles = []
|
| 554 |
+
if image.size[0] <= image_size and image.size[1] <= image_size:
|
| 555 |
+
ratio = (1, 1)
|
| 556 |
+
elif cropping:
|
| 557 |
+
tiles, ratio = dynamic_preprocess(image, min_num=min_crops, max_num=max_crops,
|
| 558 |
+
image_size=image_size)
|
| 559 |
+
else:
|
| 560 |
+
ratio = (1, 1)
|
| 561 |
+
globals_.append(transform(ImageOps.pad(image, (base_size, base_size), color=pad_color)).to(torch.bfloat16))
|
| 562 |
+
w_t, h_t = int(ratio[0]), int(ratio[1])
|
| 563 |
+
spatial.append([w_t, h_t])
|
| 564 |
+
tok = [image_token_id] * (nqb * (nqb + 1)) + [image_token_id]
|
| 565 |
+
if w_t > 1 or h_t > 1:
|
| 566 |
+
crops += [transform(c).to(torch.bfloat16) for c in tiles]
|
| 567 |
+
tok += ([image_token_id] * (nq * w_t + 1)) * (nq * h_t)
|
| 568 |
+
ids += tok
|
| 569 |
+
seq_mask += [True] * len(tok)
|
| 570 |
+
t = tokenizer.encode(splits[-1], add_special_tokens=False)
|
| 571 |
+
ids = [bos_id] + ids + t
|
| 572 |
+
seq_mask = [False] + seq_mask + [False] * len(t)
|
| 573 |
+
input_ids = torch.LongTensor(ids)
|
| 574 |
+
seq_mask = torch.tensor(seq_mask, dtype=torch.bool)
|
| 575 |
+
if n_img == 0:
|
| 576 |
+
return input_ids, None, seq_mask, None
|
| 577 |
+
images_ori = torch.stack(globals_, dim=0)
|
| 578 |
+
images_crop = (torch.stack(crops, dim=0) if crops
|
| 579 |
+
else torch.zeros((1, 3, base_size, base_size), dtype=torch.bfloat16))
|
| 580 |
+
return input_ids, (images_crop, images_ori), seq_mask, torch.tensor(spatial, dtype=torch.long)
|
| 581 |
+
|
| 582 |
+
|
| 583 |
+
def _uocr_page_count_guard(logits, gen_ids, expected_pages, page_id, eos_id, max_page_tokens, st):
|
| 584 |
+
"""vLLM unlimited_sparse_plugin.page_count_guard._PageCountGuard (per-step, in place)."""
|
| 585 |
+
if expected_pages <= 0:
|
| 586 |
+
return logits
|
| 587 |
+
n = len(gen_ids)
|
| 588 |
+
if n > st['scanned']:
|
| 589 |
+
tail = gen_ids[st['scanned']:n]
|
| 590 |
+
m = tail.count(page_id)
|
| 591 |
+
if m:
|
| 592 |
+
st['marker_cum'] += m
|
| 593 |
+
st['page_len'] = tail[::-1].index(page_id)
|
| 594 |
+
else:
|
| 595 |
+
st['page_len'] += len(tail)
|
| 596 |
+
st['scanned'] = n
|
| 597 |
+
mc = st['marker_cum']
|
| 598 |
+
if mc > expected_pages:
|
| 599 |
+
keep = logits[eos_id].clone(); logits[:] = -float('inf'); logits[eos_id] = keep
|
| 600 |
+
elif 0 < max_page_tokens <= st['page_len']:
|
| 601 |
+
tid = page_id if mc < expected_pages else eos_id
|
| 602 |
+
keep = logits[tid].clone(); logits[:] = -float('inf'); logits[tid] = keep
|
| 603 |
+
elif mc < expected_pages:
|
| 604 |
+
logits[eos_id] = -float('inf')
|
| 605 |
+
return logits
|
| 606 |
+
|
| 607 |
+
|
| 608 |
+
def _uocr_no_repeat_ngram(logits, gen_ids, ngram_size, window_size, whitelist):
|
| 609 |
+
"""vLLM deepseek_ocr.NoRepeatNGramLogitsProcessor (on generated ids only)."""
|
| 610 |
+
if ngram_size <= 0 or len(gen_ids) < ngram_size:
|
| 611 |
+
return logits
|
| 612 |
+
prefix = tuple(gen_ids[-(ngram_size - 1):]) if ngram_size > 1 else tuple()
|
| 613 |
+
start = max(0, len(gen_ids) - window_size)
|
| 614 |
+
end = len(gen_ids) - ngram_size + 1
|
| 615 |
+
banned = set()
|
| 616 |
+
for i in range(start, end):
|
| 617 |
+
ng = tuple(gen_ids[i:i + ngram_size])
|
| 618 |
+
if ng[:-1] == prefix:
|
| 619 |
+
banned.add(ng[-1])
|
| 620 |
+
banned -= whitelist
|
| 621 |
+
if banned:
|
| 622 |
+
logits[list(banned)] = -float('inf')
|
| 623 |
+
return logits
|
| 624 |
+
|
| 625 |
+
|
| 626 |
+
class UnlimitedOCRConfig(DeepseekV2Config):
|
| 627 |
+
model_type = "unlimited-ocr"
|
| 628 |
+
|
| 629 |
+
class UnlimitedOCRModel(DeepseekV2Model):
|
| 630 |
+
config_class = UnlimitedOCRConfig
|
| 631 |
+
|
| 632 |
+
def __init__(self, config: DeepseekV2Config):
|
| 633 |
+
super(UnlimitedOCRModel, self).__init__(config)
|
| 634 |
+
|
| 635 |
+
self.sam_model = build_sam_vit_b(config=config)
|
| 636 |
+
self.vision_model = build_clip_l(config=config)
|
| 637 |
+
# self.conv_2 = nn.Conv2d(in_channels=1024, out_channels=2048, kernel_size=2, stride=2)
|
| 638 |
+
n_embed = config.hidden_size
|
| 639 |
+
self.projector = MlpProjector(Dict(projector_type="linear", input_dim=config.projector_config["input_dim"], n_embed=n_embed))
|
| 640 |
+
embed_std = 1 / torch.sqrt(torch.tensor(n_embed, dtype=torch.float32))
|
| 641 |
+
self.image_newline = nn.Parameter(torch.randn(n_embed) * embed_std)
|
| 642 |
+
self.view_seperator = nn.Parameter(torch.randn(n_embed) * embed_std)
|
| 643 |
+
|
| 644 |
+
|
| 645 |
+
|
| 646 |
+
|
| 647 |
+
def forward(
|
| 648 |
+
self,
|
| 649 |
+
input_ids: torch.LongTensor = None,
|
| 650 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 651 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 652 |
+
past_key_values: Optional[List[torch.FloatTensor]] = None,
|
| 653 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
| 654 |
+
use_cache: Optional[bool] = None,
|
| 655 |
+
output_attentions: Optional[bool] = None,
|
| 656 |
+
output_hidden_states: Optional[bool] = None,
|
| 657 |
+
images: Optional[torch.FloatTensor] = None,
|
| 658 |
+
images_seq_mask: Optional[torch.FloatTensor] = None,
|
| 659 |
+
images_spatial_crop: Optional[torch.FloatTensor] = None,
|
| 660 |
+
return_dict: Optional[bool] = None,
|
| 661 |
+
) -> Union[Tuple, BaseModelOutputWithPast]:
|
| 662 |
+
|
| 663 |
+
# hand the full token id sequence directly to each sparse attention module so the
|
| 664 |
+
# indexer can derive per-page image/query ids (page-aware image bucket constraint).
|
| 665 |
+
# Set on the attention module itself (not config) to avoid any config-object identity
|
| 666 |
+
# ambiguity and to survive gradient-checkpointing recompute.
|
| 667 |
+
if input_ids is not None and getattr(input_ids, 'ndim', 0) == 2:
|
| 668 |
+
_rope2d = None
|
| 669 |
+
_layers = list(self.layers)
|
| 670 |
+
_first = next((getattr(l, 'self_attn', None) for l in _layers
|
| 671 |
+
if getattr(getattr(l, 'self_attn', None), 'indexer', None) is not None),
|
| 672 |
+
None)
|
| 673 |
+
if (_first is not None and input_ids.shape[-1] > 1
|
| 674 |
+
and getattr(_first, 'main_rope_mode', '1d') == 'page2d'):
|
| 675 |
+
_rope2d = _first.indexer._build_page_rope_pos(input_ids, input_ids.shape[-1])
|
| 676 |
+
# The per-page prefill counts (pf_cnt) that seed the decode-side page-rope state
|
| 677 |
+
# machine (_main_rope_coords) are written only on the indexer that built the
|
| 678 |
+
# coordinates (layer 0); hand them to every layer, otherwise layers 1.. fall
|
| 679 |
+
# back to 1D RoPE as soon as decoding starts.
|
| 680 |
+
_meta = (getattr(_first.indexer, '_page_rope_meta', None)
|
| 681 |
+
if (_first is not None and _rope2d is not None) else None)
|
| 682 |
+
# infer(): page-aware candidate blocks of the decode query (vLLM page_state parity).
|
| 683 |
+
_dec_allow = None
|
| 684 |
+
if _DASH_INFER["on"] and past_key_values is not None and _first is not None:
|
| 685 |
+
_dec_allow = _uocr_dash_decode_allow(_first.indexer, input_ids, past_key_values)
|
| 686 |
+
for _layer in _layers:
|
| 687 |
+
_attn = getattr(_layer, 'self_attn', None)
|
| 688 |
+
if _attn is not None and getattr(_attn, 'indexer', None) is not None:
|
| 689 |
+
_attn._sparse_source_input_ids = input_ids
|
| 690 |
+
_attn._main_rope_pos_2d = _rope2d
|
| 691 |
+
if _meta is not None:
|
| 692 |
+
_attn.indexer._page_rope_meta = _meta
|
| 693 |
+
_attn._dash_dec_allow = _dec_allow
|
| 694 |
+
|
| 695 |
+
if inputs_embeds is None:
|
| 696 |
+
# inputs_embeds = self.embed_tokens(input_ids)
|
| 697 |
+
inputs_embeds = self.get_input_embeddings()(input_ids)
|
| 698 |
+
|
| 699 |
+
|
| 700 |
+
|
| 701 |
+
sam_model = getattr(self, 'sam_model', None)
|
| 702 |
+
# sam_model = self.sam_model
|
| 703 |
+
vision_model = getattr(self, 'vision_model', None)
|
| 704 |
+
|
| 705 |
+
|
| 706 |
+
|
| 707 |
+
# TIME_DIAG=1 -> attribute the per-page vision encoder cost.
|
| 708 |
+
_t_vis = _td_now()
|
| 709 |
+
if sam_model is not None and images is not None and (input_ids.shape[1] != 1 or self.training) and torch.sum(images[0][1]).item() != 0:
|
| 710 |
+
|
| 711 |
+
_packed_mm = (inputs_embeds.shape[0] == 1 and len(images) > 1)
|
| 712 |
+
_packed_feats = []
|
| 713 |
+
for idx, image in enumerate(images):
|
| 714 |
+
images_in_this_batch = []
|
| 715 |
+
patches_all = image[0]
|
| 716 |
+
image_ori = image[1]
|
| 717 |
+
spatial = image[2] if len(image) > 2 else None
|
| 718 |
+
|
| 719 |
+
num_pages = image_ori.shape[0]
|
| 720 |
+
tile_cursor = 0
|
| 721 |
+
with torch.no_grad():
|
| 722 |
+
for page_idx in range(num_pages):
|
| 723 |
+
image_ori_p = image_ori[page_idx:page_idx + 1]
|
| 724 |
+
if spatial is not None:
|
| 725 |
+
width_crop_num = int(spatial[page_idx][0])
|
| 726 |
+
height_crop_num = int(spatial[page_idx][1])
|
| 727 |
+
else:
|
| 728 |
+
width_crop_num = height_crop_num = 1
|
| 729 |
+
do_crop = (width_crop_num > 1 or height_crop_num > 1) and torch.sum(patches_all).item() != 0
|
| 730 |
+
|
| 731 |
+
if do_crop:
|
| 732 |
+
n_tiles = width_crop_num * height_crop_num
|
| 733 |
+
patches = patches_all[tile_cursor:tile_cursor + n_tiles]
|
| 734 |
+
tile_cursor += n_tiles
|
| 735 |
+
local_features_1 = sam_model(patches)
|
| 736 |
+
local_features_2 = vision_model(patches, local_features_1)
|
| 737 |
+
local_features = torch.cat((local_features_2[:, 1:], local_features_1.flatten(2).permute(0, 2, 1)), dim=-1)
|
| 738 |
+
local_features = self.projector(local_features)
|
| 739 |
+
|
| 740 |
+
global_features_1 = sam_model(image_ori_p)
|
| 741 |
+
global_features_2 = vision_model(image_ori_p, global_features_1)
|
| 742 |
+
global_features = torch.cat((global_features_2[:, 1:], global_features_1.flatten(2).permute(0, 2, 1)), dim=-1)
|
| 743 |
+
global_features = self.projector(global_features)
|
| 744 |
+
|
| 745 |
+
_, hw, n_dim = global_features.shape
|
| 746 |
+
h = w = int(hw ** 0.5)
|
| 747 |
+
_2, hw2, n_dim2 = local_features.shape
|
| 748 |
+
h2 = w2 = int(hw2 ** 0.5)
|
| 749 |
+
|
| 750 |
+
global_features = global_features.view(h, w, n_dim)
|
| 751 |
+
global_features = torch.cat(
|
| 752 |
+
[global_features, self.image_newline[None, None, :].expand(h, 1, n_dim)], dim=1
|
| 753 |
+
)
|
| 754 |
+
global_features = global_features.view(-1, n_dim)
|
| 755 |
+
|
| 756 |
+
local_features = local_features.view(height_crop_num, width_crop_num, h2, w2, n_dim2).permute(0, 2, 1, 3, 4).reshape(height_crop_num * h2, width_crop_num * w2, n_dim2)
|
| 757 |
+
local_features = torch.cat(
|
| 758 |
+
[local_features, self.image_newline[None, None, :].expand(height_crop_num * h2, 1, n_dim2)], dim=1
|
| 759 |
+
)
|
| 760 |
+
local_features = local_features.view(-1, n_dim2)
|
| 761 |
+
|
| 762 |
+
global_local_features = torch.cat([local_features, global_features, self.view_seperator[None, :]], dim=0)
|
| 763 |
+
images_in_this_batch.append(global_local_features)
|
| 764 |
+
else:
|
| 765 |
+
global_features_1 = sam_model(image_ori_p)
|
| 766 |
+
global_features_2 = vision_model(image_ori_p, global_features_1)
|
| 767 |
+
global_features = torch.cat((global_features_2[:, 1:], global_features_1.flatten(2).permute(0, 2, 1)), dim=-1)
|
| 768 |
+
global_features = self.projector(global_features)
|
| 769 |
+
|
| 770 |
+
_, hw, n_dim = global_features.shape
|
| 771 |
+
h = w = int(hw ** 0.5)
|
| 772 |
+
global_features = global_features.view(h, w, n_dim)
|
| 773 |
+
global_features = torch.cat(
|
| 774 |
+
[global_features, self.image_newline[None, None, :].expand(h, 1, n_dim)], dim=1
|
| 775 |
+
)
|
| 776 |
+
global_features = global_features.view(-1, n_dim)
|
| 777 |
+
global_local_features = torch.cat([global_features, self.view_seperator[None, :]], dim=0)
|
| 778 |
+
images_in_this_batch.append(global_local_features)
|
| 779 |
+
|
| 780 |
+
if images_in_this_batch:
|
| 781 |
+
images_in_this_batch = torch.cat(images_in_this_batch, dim=0)
|
| 782 |
+
if _packed_mm:
|
| 783 |
+
_packed_feats.append(images_in_this_batch)
|
| 784 |
+
else:
|
| 785 |
+
inputs_embeds[idx].masked_scatter_(images_seq_mask[idx].unsqueeze(-1).cuda(), images_in_this_batch)
|
| 786 |
+
|
| 787 |
+
if _packed_mm and _packed_feats:
|
| 788 |
+
_packed_feats = torch.cat(_packed_feats, dim=0)
|
| 789 |
+
inputs_embeds[0].masked_scatter_(images_seq_mask[0].unsqueeze(-1).cuda(), _packed_feats)
|
| 790 |
+
|
| 791 |
+
_td_add("vision", _t_vis)
|
| 792 |
+
|
| 793 |
+
with _td("llm_decoder"):
|
| 794 |
+
return super(UnlimitedOCRModel, self).forward(
|
| 795 |
+
input_ids=None, attention_mask=attention_mask, past_key_values=past_key_values,
|
| 796 |
+
inputs_embeds=inputs_embeds, use_cache=use_cache, position_ids = position_ids,
|
| 797 |
+
output_attentions=output_attentions, output_hidden_states=output_hidden_states,
|
| 798 |
+
return_dict=return_dict
|
| 799 |
+
)
|
| 800 |
+
|
| 801 |
+
|
| 802 |
+
class UnlimitedOCRForCausalLM(DeepseekV2ForCausalLM):
|
| 803 |
+
|
| 804 |
+
config_class = UnlimitedOCRConfig
|
| 805 |
+
# supports_gradient_checkpointing = True
|
| 806 |
+
|
| 807 |
+
def __init__(self, config):
|
| 808 |
+
super(DeepseekV2ForCausalLM, self).__init__(config)
|
| 809 |
+
self.model = UnlimitedOCRModel(config)
|
| 810 |
+
|
| 811 |
+
self.vocab_size = config.vocab_size
|
| 812 |
+
|
| 813 |
+
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
| 814 |
+
|
| 815 |
+
# self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
| 816 |
+
|
| 817 |
+
# Initialize weights and apply final processing
|
| 818 |
+
self.post_init()
|
| 819 |
+
|
| 820 |
+
def get_model(self):
|
| 821 |
+
return self.model
|
| 822 |
+
|
| 823 |
+
|
| 824 |
+
def forward(
|
| 825 |
+
self,
|
| 826 |
+
input_ids: torch.LongTensor = None,
|
| 827 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 828 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 829 |
+
past_key_values: Optional[List[torch.FloatTensor]] = None,
|
| 830 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
| 831 |
+
labels: Optional[torch.LongTensor] = None,
|
| 832 |
+
use_cache: Optional[bool] = None,
|
| 833 |
+
output_attentions: Optional[bool] = None,
|
| 834 |
+
output_hidden_states: Optional[bool] = None,
|
| 835 |
+
images: Optional[torch.FloatTensor] = None,
|
| 836 |
+
images_seq_mask: Optional[torch.FloatTensor] = None,
|
| 837 |
+
images_spatial_crop: Optional[torch.FloatTensor] = None,
|
| 838 |
+
return_dict: Optional[bool] = None,
|
| 839 |
+
|
| 840 |
+
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 841 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 842 |
+
output_hidden_states = (
|
| 843 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 844 |
+
)
|
| 845 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 846 |
+
|
| 847 |
+
|
| 848 |
+
|
| 849 |
+
outputs = self.model(
|
| 850 |
+
input_ids=input_ids,
|
| 851 |
+
past_key_values=past_key_values,
|
| 852 |
+
attention_mask=attention_mask,
|
| 853 |
+
position_ids=position_ids,
|
| 854 |
+
inputs_embeds=inputs_embeds,
|
| 855 |
+
use_cache=use_cache,
|
| 856 |
+
output_attentions=output_attentions,
|
| 857 |
+
output_hidden_states=output_hidden_states,
|
| 858 |
+
images=images,
|
| 859 |
+
images_seq_mask = images_seq_mask,
|
| 860 |
+
images_spatial_crop = images_spatial_crop,
|
| 861 |
+
return_dict=return_dict
|
| 862 |
+
|
| 863 |
+
)
|
| 864 |
+
|
| 865 |
+
|
| 866 |
+
|
| 867 |
+
# print(transformer_outputs)
|
| 868 |
+
|
| 869 |
+
hidden_states = outputs[0]
|
| 870 |
+
|
| 871 |
+
# fused chunked linear cross-entropy. Avoids materialising the full
|
| 872 |
+
# [B, seq, vocab=129280] logits (and its fp32 upcast) that OOMs on long OCR sequences.
|
| 873 |
+
# Gated by FUSED_CHUNK_CE (default on when labels present); falls back to the classic
|
| 874 |
+
# full-logits path so generation / return-logits callers stay unaffected.
|
| 875 |
+
_use_fused_ce = (
|
| 876 |
+
labels is not None
|
| 877 |
+
and os.environ.get('FUSED_CHUNK_CE', '1') == '1'
|
| 878 |
+
)
|
| 879 |
+
|
| 880 |
+
logits = None
|
| 881 |
+
loss = None
|
| 882 |
+
token_acc_correct = None
|
| 883 |
+
token_acc_valid = None
|
| 884 |
+
if _use_fused_ce:
|
| 885 |
+
with _td("ce"):
|
| 886 |
+
loss, token_acc_correct, token_acc_valid = _fused_chunked_lm_ce(
|
| 887 |
+
hidden_states,
|
| 888 |
+
self.lm_head,
|
| 889 |
+
labels,
|
| 890 |
+
self.config.vocab_size,
|
| 891 |
+
num_items_in_batch=None,
|
| 892 |
+
chunk_size=int(os.environ.get('CE_CHUNK_SIZE', '1024')),
|
| 893 |
+
return_token_acc=True,
|
| 894 |
+
)
|
| 895 |
+
_td_report(f"seq={hidden_states.shape[1]}")
|
| 896 |
+
else:
|
| 897 |
+
logits = self.lm_head(hidden_states)
|
| 898 |
+
logits = logits.float()
|
| 899 |
+
|
| 900 |
+
if labels is not None:
|
| 901 |
+
# Shift so that tokens < n predict n
|
| 902 |
+
shift_logits = logits[..., :-1, :].contiguous()
|
| 903 |
+
shift_labels = labels[..., 1:].contiguous()
|
| 904 |
+
# Flatten the tokens
|
| 905 |
+
loss_fct = CrossEntropyLoss()
|
| 906 |
+
shift_logits = shift_logits.view(-1, self.config.vocab_size)
|
| 907 |
+
shift_labels = shift_labels.view(-1)
|
| 908 |
+
# Enable model parallelism
|
| 909 |
+
shift_labels = shift_labels.to(shift_logits.device)
|
| 910 |
+
loss = loss_fct(shift_logits, shift_labels)
|
| 911 |
+
|
| 912 |
+
# propagate indexer_kl_loss to outer output; trainer is responsible for
|
| 913 |
+
# weighting/adding it into the final loss and logging. Do NOT add KL here to
|
| 914 |
+
# avoid double-counting when swift's compute_loss picks it up.
|
| 915 |
+
_kl = getattr(outputs, 'indexer_kl_loss', None)
|
| 916 |
+
# one-shot diagnostic
|
| 917 |
+
if not getattr(self, '_kl_prop_debug_logged', False):
|
| 918 |
+
self._kl_prop_debug_logged = True
|
| 919 |
+
try:
|
| 920 |
+
_inner_type = type(outputs).__name__
|
| 921 |
+
_kl_repr = 'None' if _kl is None else f'tensor(shape={list(_kl.shape)}, dtype={_kl.dtype}, val={float(_kl.detach()):.6f})'
|
| 922 |
+
print(f"[debug ForCausalLM] inner_outputs_type={_inner_type} "
|
| 923 |
+
f"inner_has_indexer_kl={hasattr(outputs, 'indexer_kl_loss')} "
|
| 924 |
+
f"_kl={_kl_repr} labels_not_None={labels is not None}", flush=True)
|
| 925 |
+
except Exception as _e:
|
| 926 |
+
print(f"[debug ForCausalLM] failed: {_e}", flush=True)
|
| 927 |
+
|
| 928 |
+
if not return_dict:
|
| 929 |
+
output = (logits,) + outputs[1:]
|
| 930 |
+
return (loss,) + output if loss is not None else output
|
| 931 |
+
|
| 932 |
+
result = CausalLMOutputWithPastAndKL(
|
| 933 |
+
loss=loss,
|
| 934 |
+
logits=logits,
|
| 935 |
+
past_key_values=outputs.past_key_values,
|
| 936 |
+
hidden_states=outputs.hidden_states,
|
| 937 |
+
attentions=outputs.attentions,
|
| 938 |
+
indexer_kl_loss=_kl, # proper dataclass field, survives DDP pytree
|
| 939 |
+
token_acc_correct=token_acc_correct, # fused-CE correct token count
|
| 940 |
+
token_acc_valid=token_acc_valid, # fused-CE valid token count
|
| 941 |
+
)
|
| 942 |
+
return result
|
| 943 |
+
|
| 944 |
+
|
| 945 |
+
def prepare_inputs_for_generation(
|
| 946 |
+
self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
|
| 947 |
+
):
|
| 948 |
+
# Omit tokens covered by past_key_values
|
| 949 |
+
past_length = 0
|
| 950 |
+
if past_key_values is not None:
|
| 951 |
+
if isinstance(past_key_values, Cache):
|
| 952 |
+
cache_length = past_key_values.get_seq_length()
|
| 953 |
+
past_length = past_key_values.get_seq_length()
|
| 954 |
+
max_cache_length = getattr(past_key_values, 'get_max_length', lambda: None)()
|
| 955 |
+
else:
|
| 956 |
+
cache_length = past_length = past_key_values[0][0].shape[2]
|
| 957 |
+
max_cache_length = None
|
| 958 |
+
|
| 959 |
+
# Ring buffer: cache size is fixed, but we've processed more tokens.
|
| 960 |
+
# Always just take the last token for decode.
|
| 961 |
+
if hasattr(past_key_values, '_prefill_length') and past_length > 0:
|
| 962 |
+
input_ids = input_ids[:, -1:]
|
| 963 |
+
# Keep only the unprocessed tokens:
|
| 964 |
+
# 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
|
| 965 |
+
# some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as
|
| 966 |
+
# input)
|
| 967 |
+
elif attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
|
| 968 |
+
input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
|
| 969 |
+
# 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
|
| 970 |
+
# input_ids based on the past_length.
|
| 971 |
+
elif past_length < input_ids.shape[1]:
|
| 972 |
+
input_ids = input_ids[:, past_length:]
|
| 973 |
+
# 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
|
| 974 |
+
|
| 975 |
+
# If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
|
| 976 |
+
if (
|
| 977 |
+
max_cache_length is not None
|
| 978 |
+
and attention_mask is not None
|
| 979 |
+
and cache_length + input_ids.shape[1] > max_cache_length
|
| 980 |
+
):
|
| 981 |
+
attention_mask = attention_mask[:, -max_cache_length:]
|
| 982 |
+
|
| 983 |
+
position_ids = kwargs.get("position_ids", None)
|
| 984 |
+
if attention_mask is not None and position_ids is None:
|
| 985 |
+
# create position_ids on the fly for batch generation
|
| 986 |
+
position_ids = attention_mask.long().cumsum(-1) - 1
|
| 987 |
+
position_ids.masked_fill_(attention_mask == 0, 1)
|
| 988 |
+
if past_key_values:
|
| 989 |
+
position_ids = position_ids[:, -input_ids.shape[1] :]
|
| 990 |
+
|
| 991 |
+
# if self.generation_config.cache_implementation == "static":
|
| 992 |
+
# # generation with static cache
|
| 993 |
+
# cache_position = kwargs.get("cache_position", None)
|
| 994 |
+
# if cache_position is None:
|
| 995 |
+
# past_length = 0
|
| 996 |
+
# else:
|
| 997 |
+
# past_length = cache_position[-1] + 1
|
| 998 |
+
# input_ids = input_ids[:, past_length:]
|
| 999 |
+
# position_ids = position_ids[:, past_length:]
|
| 1000 |
+
|
| 1001 |
+
# TODO @gante we should only keep a `cache_position` in generate, and do +=1.
|
| 1002 |
+
# same goes for position ids. Could also help with continued generation.
|
| 1003 |
+
cache_position = torch.arange(past_length, past_length + position_ids.shape[-1], device=position_ids.device)
|
| 1004 |
+
|
| 1005 |
+
# if `inputs_embeds` are passed, we only want to use them in the 1st generation step
|
| 1006 |
+
if inputs_embeds is not None and past_key_values is None:
|
| 1007 |
+
model_inputs = {"inputs_embeds": inputs_embeds}
|
| 1008 |
+
else:
|
| 1009 |
+
model_inputs = {"input_ids": input_ids}
|
| 1010 |
+
|
| 1011 |
+
# Pass images only on prefill (cache empty or None)
|
| 1012 |
+
_is_prefill = (past_key_values is None or
|
| 1013 |
+
(isinstance(past_key_values, Cache) and past_key_values.get_seq_length() == 0))
|
| 1014 |
+
model_inputs.update(
|
| 1015 |
+
{
|
| 1016 |
+
"position_ids": position_ids,
|
| 1017 |
+
"past_key_values": past_key_values,
|
| 1018 |
+
"use_cache": kwargs.get("use_cache"),
|
| 1019 |
+
"attention_mask": attention_mask,
|
| 1020 |
+
"images": kwargs.get("images", None) if _is_prefill else None,
|
| 1021 |
+
"images_seq_mask": kwargs.get("images_seq_mask", None) if _is_prefill else None,
|
| 1022 |
+
"images_spatial_crop": kwargs.get("images_spatial_crop", None) if _is_prefill else None,
|
| 1023 |
+
}
|
| 1024 |
+
)
|
| 1025 |
+
return model_inputs
|
| 1026 |
+
|
| 1027 |
+
|
| 1028 |
+
def disable_torch_init(self):
|
| 1029 |
+
"""
|
| 1030 |
+
Disable the redundant torch default initialization to accelerate model creation.
|
| 1031 |
+
"""
|
| 1032 |
+
import torch
|
| 1033 |
+
setattr(torch.nn.Linear, "reset_parameters", lambda self: None)
|
| 1034 |
+
setattr(torch.nn.LayerNorm, "reset_parameters", lambda self: None)
|
| 1035 |
+
|
| 1036 |
+
|
| 1037 |
+
|
| 1038 |
+
@torch.no_grad()
|
| 1039 |
+
def infer(self, tokenizer, prompt='', image_file='', output_path='', base_size=1024, image_size=640,
|
| 1040 |
+
crop_mode=None, test_compress=False, save_results=False, eval_mode=False, max_length=131072,
|
| 1041 |
+
max_new_tokens=None, tps_interval=0, no_repeat_ngram_size=0, ngram_window=0,
|
| 1042 |
+
ngram_whitelist_token_ids=None, temperature=0.0, page_count_guard=True,
|
| 1043 |
+
max_page_tokens=20000, dash_topm=16, max_crops=16, return_dict=False):
|
| 1044 |
+
"""Single-/multi-page inference, aligned with the vLLM deployment
|
| 1045 |
+
(sparse_attn_infer-451/deploy_multimodel_dash_01.sh + request_openai*.py).
|
| 1046 |
+
|
| 1047 |
+
image_file: one path / PIL image, or a list of them (one per page, in order).
|
| 1048 |
+
prompt: instruction text. Without ``<image>`` placeholders the trained multi-page
|
| 1049 |
+
layout is built: ``"<PAGE><image>" * n_pages + prompt`` (default prompt
|
| 1050 |
+
"Multi page parsing."). A prompt with exactly n_pages ``<image>`` is used as is;
|
| 1051 |
+
a single ``<image>`` with several pages is expanded to ``<PAGE><image>`` per page.
|
| 1052 |
+
crop_mode: None -> vLLM auto (gundam tiles for one page, base for several);
|
| 1053 |
+
True / False forces gundam / base.
|
| 1054 |
+
Generation mirrors vLLM: greedy when temperature == 0, stops at EOS or
|
| 1055 |
+
``max_new_tokens`` (default ``max_length - prompt_len``), PageCountGuard on by default
|
| 1056 |
+
(prompt <PAGE> count == output pages; per-page cap ``max_page_tokens``), optional
|
| 1057 |
+
NoRepeatNGram (``no_repeat_ngram_size`` / ``ngram_window``, vLLM extra_args semantics).
|
| 1058 |
+
Decoder attention = DASH block-sparse in prefill and decode (vLLM DASH_DECODE_DENSE=0,
|
| 1059 |
+
DASH_INFER_TOPM=dash_topm, DASH_PAGE_VISIBLE_BACK=0).
|
| 1060 |
+
|
| 1061 |
+
Returns the generated text (special tokens kept, EOS dropped -- like vLLM with
|
| 1062 |
+
skip_special_tokens=False), or a dict with token ids when ``return_dict``.
|
| 1063 |
+
"""
|
| 1064 |
+
if isinstance(image_file, (list, tuple)):
|
| 1065 |
+
files = [f for f in image_file if f is not None and f != '']
|
| 1066 |
+
elif image_file is None or image_file == '':
|
| 1067 |
+
files = []
|
| 1068 |
+
else:
|
| 1069 |
+
files = [image_file]
|
| 1070 |
+
images = [f.convert('RGB') if isinstance(f, Image.Image) else Image.open(f).convert('RGB')
|
| 1071 |
+
for f in files]
|
| 1072 |
+
n_img = len(images)
|
| 1073 |
+
prompt = prompt or ('Multi page parsing.' if n_img else '')
|
| 1074 |
+
if not prompt:
|
| 1075 |
+
raise ValueError('prompt is empty!')
|
| 1076 |
+
n_ph = prompt.count('<image>')
|
| 1077 |
+
if n_img and n_ph == 0:
|
| 1078 |
+
prompt = '<PAGE><image>' * n_img + prompt
|
| 1079 |
+
elif n_img > 1 and n_ph == 1:
|
| 1080 |
+
prompt = prompt.replace('<image>', '<PAGE><image>' * n_img)
|
| 1081 |
+
elif n_ph != n_img:
|
| 1082 |
+
raise ValueError(f'prompt has {n_ph} <image> placeholders but {n_img} images')
|
| 1083 |
+
|
| 1084 |
+
input_ids, imgs, seq_mask, spatial = _uocr_build_inputs(
|
| 1085 |
+
tokenizer, prompt, images, crop_mode=crop_mode, base_size=base_size,
|
| 1086 |
+
image_size=image_size, max_crops=max_crops)
|
| 1087 |
+
res = self._uocr_generate(
|
| 1088 |
+
tokenizer, input_ids, imgs, seq_mask, spatial,
|
| 1089 |
+
max_new_tokens=(max_new_tokens if max_new_tokens else max(1, max_length - input_ids.numel())),
|
| 1090 |
+
max_length=max_length, temperature=temperature, page_count_guard=page_count_guard,
|
| 1091 |
+
max_page_tokens=max_page_tokens, ngram_size=no_repeat_ngram_size,
|
| 1092 |
+
ngram_window=(ngram_window if ngram_window > 0 else 100),
|
| 1093 |
+
ngram_whitelist=set(ngram_whitelist_token_ids or []), dash_topm=dash_topm,
|
| 1094 |
+
stream=not eval_mode, tps_interval=tps_interval)
|
| 1095 |
+
outputs = res['text']
|
| 1096 |
+
if test_compress and n_img:
|
| 1097 |
+
out_len = len(tokenizer.encode(outputs, add_special_tokens=False))
|
| 1098 |
+
n_vis = int(seq_mask.sum())
|
| 1099 |
+
print('=' * 50)
|
| 1100 |
+
print('image tokens: ', n_vis, ' output tokens: ', out_len,
|
| 1101 |
+
' compression ratio: ', round(out_len / max(1, n_vis), 2))
|
| 1102 |
+
print('=' * 50)
|
| 1103 |
+
if save_results and output_path:
|
| 1104 |
+
self._uocr_save_results(outputs, images, output_path)
|
| 1105 |
+
if return_dict:
|
| 1106 |
+
res['prompt'] = prompt
|
| 1107 |
+
return res
|
| 1108 |
+
return outputs
|
| 1109 |
+
|
| 1110 |
+
def infer_multi(self, tokenizer, prompt='', image_files=None, output_path='', image_size=640,
|
| 1111 |
+
save_results=False, max_length=131072, tps_interval=0, no_repeat_ngram_size=0,
|
| 1112 |
+
ngram_window=0, temperature=0.0, **kwargs):
|
| 1113 |
+
"""Backward-compatible multi-page entry point -> ``infer`` (base mode, no crop).
|
| 1114 |
+
Returns (outputs, output_tokens) like before."""
|
| 1115 |
+
if not image_files:
|
| 1116 |
+
raise ValueError('image_files must be a non-empty list for multi-image inference!')
|
| 1117 |
+
kwargs.setdefault('crop_mode', False)
|
| 1118 |
+
res = self.infer(tokenizer, prompt=prompt.replace('<image>', '') if prompt.count('<image>') == 1 else prompt,
|
| 1119 |
+
image_file=list(image_files), output_path=output_path, image_size=image_size,
|
| 1120 |
+
save_results=save_results, max_length=max_length, tps_interval=tps_interval,
|
| 1121 |
+
no_repeat_ngram_size=no_repeat_ngram_size, ngram_window=ngram_window,
|
| 1122 |
+
temperature=temperature, return_dict=True, **kwargs)
|
| 1123 |
+
return res['text'], len(res['token_ids'])
|
| 1124 |
+
|
| 1125 |
+
@torch.no_grad()
|
| 1126 |
+
def _uocr_generate(self, tokenizer, input_ids, imgs, seq_mask, spatial, max_new_tokens=16384,
|
| 1127 |
+
max_length=131072, temperature=0.0, page_count_guard=True, max_page_tokens=20000,
|
| 1128 |
+
ngram_size=0, ngram_window=100, ngram_whitelist=None, dash_topm=16,
|
| 1129 |
+
stream=False, tps_interval=0):
|
| 1130 |
+
"""Prefill + token-by-token decode on a DynamicCache (no HF generate()).
|
| 1131 |
+
Only the last position goes through lm_head (full-sequence logits at 30k tokens x
|
| 1132 |
+
129k vocab would not fit)."""
|
| 1133 |
+
import time
|
| 1134 |
+
from transformers.cache_utils import DynamicCache
|
| 1135 |
+
device = self.lm_head.weight.device
|
| 1136 |
+
ids = input_ids.unsqueeze(0).to(device)
|
| 1137 |
+
prompt_ids = input_ids.tolist()
|
| 1138 |
+
page_id = int((self.config.sparse_attention_config or {}).get('page_token_id', 128827)) \
|
| 1139 |
+
if isinstance(getattr(self.config, 'sparse_attention_config', None), dict) else 128827
|
| 1140 |
+
eos_id = int(tokenizer.eos_token_id)
|
| 1141 |
+
expected_pages = prompt_ids.count(page_id) if page_count_guard else 0
|
| 1142 |
+
guard_st = {'marker_cum': 0, 'scanned': 0, 'page_len': 0}
|
| 1143 |
+
kw = {}
|
| 1144 |
+
if imgs is not None:
|
| 1145 |
+
kw = dict(images=[(imgs[0].to(device), imgs[1].to(device), spatial)],
|
| 1146 |
+
images_seq_mask=seq_mask.unsqueeze(0).to(device),
|
| 1147 |
+
images_spatial_crop=spatial)
|
| 1148 |
+
|
| 1149 |
+
# decode-time knobs: no ring-buffer SWA (vLLM attends via DASH), vLLM-parity router
|
| 1150 |
+
cfg = self.config
|
| 1151 |
+
saved_cfg = (getattr(cfg, '_ring_window', None), getattr(cfg, 'sliding_window', None))
|
| 1152 |
+
cfg._ring_window = None
|
| 1153 |
+
cfg.sliding_window = None
|
| 1154 |
+
dash_state = configure_dash_inference(self.model, True, topm=dash_topm)
|
| 1155 |
+
streamer = TextStreamer(tokenizer, skip_prompt=False, skip_special_tokens=False) if stream else None
|
| 1156 |
+
gen, finish = [], 'length'
|
| 1157 |
+
t0 = time.time()
|
| 1158 |
+
t_first = None
|
| 1159 |
+
try:
|
| 1160 |
+
cache = DynamicCache()
|
| 1161 |
+
S = ids.shape[1]
|
| 1162 |
+
out = self.model(input_ids=ids, past_key_values=cache, use_cache=True,
|
| 1163 |
+
position_ids=torch.arange(S, device=device)[None], return_dict=True, **kw)
|
| 1164 |
+
cache = out.past_key_values
|
| 1165 |
+
h = out.last_hidden_state[:, -1:]
|
| 1166 |
+
for step in range(max_new_tokens):
|
| 1167 |
+
logits = self.lm_head(h).float()[0, -1]
|
| 1168 |
+
if ngram_size > 0:
|
| 1169 |
+
logits = _uocr_no_repeat_ngram(logits, gen, ngram_size, ngram_window, ngram_whitelist or set())
|
| 1170 |
+
if expected_pages > 0:
|
| 1171 |
+
logits = _uocr_page_count_guard(logits, gen, expected_pages, page_id, eos_id,
|
| 1172 |
+
max_page_tokens, guard_st)
|
| 1173 |
+
if temperature and temperature > 0:
|
| 1174 |
+
nxt = int(torch.multinomial(torch.softmax(logits / temperature, dim=-1), 1))
|
| 1175 |
+
else:
|
| 1176 |
+
nxt = int(torch.argmax(logits))
|
| 1177 |
+
if t_first is None:
|
| 1178 |
+
t_first = time.time()
|
| 1179 |
+
if nxt == eos_id:
|
| 1180 |
+
finish = 'stop'
|
| 1181 |
+
break
|
| 1182 |
+
gen.append(nxt)
|
| 1183 |
+
if streamer is not None:
|
| 1184 |
+
streamer.put(torch.tensor([nxt]))
|
| 1185 |
+
if tps_interval and len(gen) % tps_interval == 0:
|
| 1186 |
+
dt = time.time() - t_first
|
| 1187 |
+
print(f'\n[TPS] tokens={len(gen)} avg={len(gen) / max(dt, 1e-6):.1f} t/s', flush=True)
|
| 1188 |
+
pos = S + len(gen) - 1
|
| 1189 |
+
if pos + 1 >= max_length:
|
| 1190 |
+
break
|
| 1191 |
+
out = self.model(input_ids=torch.tensor([[nxt]], device=device), past_key_values=cache,
|
| 1192 |
+
use_cache=True, position_ids=torch.tensor([[pos]], device=device),
|
| 1193 |
+
return_dict=True)
|
| 1194 |
+
cache = out.past_key_values
|
| 1195 |
+
h = out.last_hidden_state
|
| 1196 |
+
finally:
|
| 1197 |
+
configure_dash_inference(self.model, False, state=dash_state)
|
| 1198 |
+
cfg._ring_window, cfg.sliding_window = saved_cfg
|
| 1199 |
+
if streamer is not None:
|
| 1200 |
+
streamer.end()
|
| 1201 |
+
text = tokenizer.decode(gen, skip_special_tokens=False)
|
| 1202 |
+
return {'text': text, 'token_ids': gen, 'prompt_token_ids': prompt_ids,
|
| 1203 |
+
'finish_reason': finish, 'num_pages_out': gen.count(page_id),
|
| 1204 |
+
'expected_pages': expected_pages, 'time': time.time() - t0,
|
| 1205 |
+
'prefill_time': (t_first - t0) if t_first else None}
|
| 1206 |
+
|
| 1207 |
+
def _uocr_save_results(self, outputs, images, output_path):
|
| 1208 |
+
os.makedirs(output_path, exist_ok=True)
|
| 1209 |
+
os.makedirs(f'{output_path}/images', exist_ok=True)
|
| 1210 |
+
pages = outputs.split('<PAGE>')[1:] if '<PAGE>' in outputs else [outputs]
|
| 1211 |
+
processed = []
|
| 1212 |
+
for page_idx, page_output in enumerate(pages):
|
| 1213 |
+
page_output = page_output.strip()
|
| 1214 |
+
if page_idx < len(images):
|
| 1215 |
+
matches_ref, matches_images, mathes_other = re_match(page_output)
|
| 1216 |
+
prefix = f'page_{page_idx}_'
|
| 1217 |
+
result = process_image_with_refs(images[page_idx].copy(), matches_ref, output_path,
|
| 1218 |
+
image_prefix=prefix)
|
| 1219 |
+
result.save(f'{output_path}/result_with_boxes_{page_idx}.jpg')
|
| 1220 |
+
for idx, m in enumerate(matches_images):
|
| 1221 |
+
page_output = page_output.replace(m, f'\n')
|
| 1222 |
+
for m in mathes_other:
|
| 1223 |
+
page_output = page_output.replace(m, '').replace('\\coloneqq', ':=').replace('\\eqqcolon', '=:')
|
| 1224 |
+
processed.append(page_output)
|
| 1225 |
+
with open(f'{output_path}/result.md', 'w', encoding='utf-8') as f:
|
| 1226 |
+
f.write('<PAGE>\n' + '\n<PAGE>\n'.join(processed))
|
| 1227 |
+
with open(f'{output_path}/result_raw.md', 'w', encoding='utf-8') as f:
|
| 1228 |
+
f.write(outputs)
|
packing_patch.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Sequence-packing support for the sparse-attention path.
|
| 2 |
+
|
| 3 |
+
Ported from Megatron-Bridge (``deepseek_ocr2/modeling_deepseek_ocr_mcore.py``):
|
| 4 |
+
the two-stage warmup/sparse pipeline there runs with packing, and does so by
|
| 5 |
+
making every causal / local-window / block mask *segment aware* instead of
|
| 6 |
+
assuming a single contiguous sequence per row.
|
| 7 |
+
|
| 8 |
+
Why this is needed
|
| 9 |
+
-------------------
|
| 10 |
+
With packing, ms-swift concatenates several samples into one row and emits
|
| 11 |
+
per-sample-RESET ``position_ids`` (``[0,1,..,L1-1, 0,1,..,L2-1, ...]``; see
|
| 12 |
+
``swift/template/base.py::packing_row``). The sparse code compares the query
|
| 13 |
+
``position_ids`` against ``k_positions = arange(k_len)`` (the GLOBAL flat key
|
| 14 |
+
index) to build causal / local masks. With reset positions that comparison is
|
| 15 |
+
wrong (a sample-2 query at local pos 0 would be "allowed" to see only key 0,
|
| 16 |
+
which belongs to sample 1), so packing silently corrupts attention.
|
| 17 |
+
|
| 18 |
+
Fix (mirrors Megatron-Bridge)
|
| 19 |
+
-----------------------------
|
| 20 |
+
* Derive a per-token SEGMENT id from the ``position_ids`` resets.
|
| 21 |
+
* Evaluate causal / local windows in GLOBAL key space using the flat query
|
| 22 |
+
index (for a packed row with batch==1 the flat index *is* the global
|
| 23 |
+
position, and ``k_positions`` is already global).
|
| 24 |
+
* AND every keep-mask with a ``same_segment`` constraint so no query attends
|
| 25 |
+
across a sample boundary.
|
| 26 |
+
* Bypass the fused Triton kernels when packed (they assume ``position_ids ==
|
| 27 |
+
arange`` and a single segment); the eager PyTorch paths below are made
|
| 28 |
+
segment aware instead.
|
| 29 |
+
|
| 30 |
+
batch==1 is assumed for packed rows (``padding_free``/packing collapses the
|
| 31 |
+
batch dimension into the sequence).
|
| 32 |
+
|
| 33 |
+
Block-grid alignment (sparse stage)
|
| 34 |
+
-----------------------------------
|
| 35 |
+
Key blocks are cut on the FLAT (global) token index, so a sample that does not
|
| 36 |
+
start on a ``block_size`` boundary gets a block grid shifted by
|
| 37 |
+
``start % block_size`` relative to single-sample inference. That is safe (the
|
| 38 |
+
segment masks above still prevent any cross-sample attention) but it is a
|
| 39 |
+
train/inference discrepancy in *how the keys are grouped*, plus the first block
|
| 40 |
+
of each sample straddles the previous one and gets trimmed. Run packing with
|
| 41 |
+
``PACKING_ALIGN_SIZE=<block_size>`` (ms-swift ``Template._align_packing_row``)
|
| 42 |
+
to make every sample start on a block boundary; ``check_block_alignment`` warns
|
| 43 |
+
once when it is not the case.
|
| 44 |
+
"""
|
| 45 |
+
import torch
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def is_packed(position_ids):
|
| 49 |
+
"""A packed row restarts ``position_ids`` at 0 at each sample boundary, so a
|
| 50 |
+
0 appears at some column >= 1. A plain (non-packed) sequence only has its
|
| 51 |
+
single 0 at column 0. Robust for training (no left padding under
|
| 52 |
+
padding_free)."""
|
| 53 |
+
if position_ids is None or position_ids.dim() != 2 or position_ids.shape[1] < 2:
|
| 54 |
+
return False
|
| 55 |
+
return bool((position_ids[:, 1:] == 0).any())
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def segment_ids(position_ids):
|
| 59 |
+
"""[B, S] long: 0-based segment index of every token, from position_ids
|
| 60 |
+
resets. ``(position_ids == 0)`` marks each segment start; cumsum-1 numbers
|
| 61 |
+
them. For a non-packed row this is all zeros (single segment)."""
|
| 62 |
+
return (position_ids == 0).to(torch.long).cumsum(dim=1) - 1
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def packed_masks(position_ids, k_positions, q_index, local_window):
|
| 66 |
+
"""All masks are built in GLOBAL key space and are [B, 1, C, Sk] bool.
|
| 67 |
+
|
| 68 |
+
Args:
|
| 69 |
+
position_ids: [B, S] reset positions (training prefill: S == k_len).
|
| 70 |
+
k_positions: [Sk] global key indices == ``arange(k_len)``.
|
| 71 |
+
q_index: [C] global (flat) indices of the query rows in this
|
| 72 |
+
chunk (e.g. ``arange(c0, c1)`` or the strided
|
| 73 |
+
``sel`` used by the KL loss). These ARE the global
|
| 74 |
+
query positions for a packed batch==1 row.
|
| 75 |
+
local_window: int inclusive local window length in tokens.
|
| 76 |
+
|
| 77 |
+
Returns dict with:
|
| 78 |
+
token_future [B,1,C,Sk] key is strictly after the query (to mask out)
|
| 79 |
+
local_token [B,1,C,Sk] key in [gpos-local_window+1, gpos]
|
| 80 |
+
same_seg [B,1,C,Sk] key and query share a segment
|
| 81 |
+
gpos [B,C] global query positions
|
| 82 |
+
"""
|
| 83 |
+
device = position_ids.device
|
| 84 |
+
B = position_ids.shape[0]
|
| 85 |
+
Sk = k_positions.shape[0]
|
| 86 |
+
seg = segment_ids(position_ids) # [B, S]
|
| 87 |
+
seg_k = seg[:, :Sk] # [B, Sk]
|
| 88 |
+
q_index = q_index.to(device=device, dtype=torch.long)
|
| 89 |
+
seg_q = seg.index_select(1, q_index) # [B, C]
|
| 90 |
+
gpos = q_index.view(1, -1).expand(B, -1) # [B, C] global q pos
|
| 91 |
+
|
| 92 |
+
kp = k_positions.view(1, 1, 1, -1) # [1,1,1,Sk]
|
| 93 |
+
gpos_e = gpos[:, None, :, None] # [B,1,C,1]
|
| 94 |
+
token_future = kp > gpos_e
|
| 95 |
+
ls = (gpos - (local_window - 1)).clamp(min=0)[:, None, :, None]
|
| 96 |
+
local_token = (kp >= ls) & (kp <= gpos_e)
|
| 97 |
+
same_seg = seg_k[:, None, None, :] == seg_q[:, None, :, None]
|
| 98 |
+
return {
|
| 99 |
+
'token_future': token_future,
|
| 100 |
+
'local_token': local_token,
|
| 101 |
+
'same_seg': same_seg,
|
| 102 |
+
'gpos': gpos,
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def segment_of(position_ids, token_indices):
|
| 107 |
+
"""Gather the segment id for arbitrary GLOBAL token indices.
|
| 108 |
+
|
| 109 |
+
Args:
|
| 110 |
+
position_ids: [B, S]
|
| 111 |
+
token_indices: [B, ...] long, global key indices (clamped by caller).
|
| 112 |
+
Returns tensor shaped like ``token_indices`` with the segment id of each.
|
| 113 |
+
"""
|
| 114 |
+
seg = segment_ids(position_ids) # [B, S]
|
| 115 |
+
flat = token_indices.reshape(token_indices.shape[0], -1)
|
| 116 |
+
out = seg.gather(1, flat.clamp(min=0, max=seg.shape[1] - 1))
|
| 117 |
+
return out.reshape(token_indices.shape)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def segment_starts(position_ids):
|
| 121 |
+
"""[N] long: GLOBAL start index of every segment in a packed row (batch 0).
|
| 122 |
+
``start = flat_index - position_ids`` at the reset points."""
|
| 123 |
+
return (position_ids[0] == 0).nonzero(as_tuple=True)[0]
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
_ALIGN_WARNED = False
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def check_block_alignment(position_ids, block_size):
|
| 130 |
+
"""one-shot warning when a packed row's samples do NOT start on a
|
| 131 |
+
``block_size`` boundary.
|
| 132 |
+
|
| 133 |
+
The sparse stage cuts its key blocks on the FLAT (global) token index, so a sample
|
| 134 |
+
starting at ``s`` gets a block grid shifted by ``s % block_size`` relative to
|
| 135 |
+
single-sample inference (which starts at 0), and its first block straddles the previous
|
| 136 |
+
sample (the segment mask then trims it, wasting part of the remote top-k budget).
|
| 137 |
+
Correctness is unaffected (no cross-sample leakage), but the train/inference block
|
| 138 |
+
partitioning differs. Set ``PACKING_ALIGN_SIZE=<block_size>`` so ms-swift's
|
| 139 |
+
``packing_row`` pads every sample up to that multiple and the two grids coincide.
|
| 140 |
+
"""
|
| 141 |
+
global _ALIGN_WARNED
|
| 142 |
+
if _ALIGN_WARNED or position_ids is None or block_size <= 1:
|
| 143 |
+
return
|
| 144 |
+
starts = segment_starts(position_ids)
|
| 145 |
+
n_bad = int((starts % block_size != 0).sum())
|
| 146 |
+
if n_bad:
|
| 147 |
+
_ALIGN_WARNED = True
|
| 148 |
+
print(f'[PackingAlign] {n_bad}/{starts.numel()} packed samples do not start on a '
|
| 149 |
+
f'{block_size}-token block boundary -> block grid is shifted w.r.t. '
|
| 150 |
+
f'single-sample inference. Set PACKING_ALIGN_SIZE={block_size} to align.',
|
| 151 |
+
flush=True)
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def dense_segment_keep(position_ids, device):
|
| 155 |
+
"""Boolean [B, 1, S, S] keep-mask for the DENSE (warmup) main branch:
|
| 156 |
+
True where a query may attend (same segment AND causal). Passed straight to
|
| 157 |
+
``scaled_dot_product_attention(attn_mask=...)`` (bool: True == attend).
|
| 158 |
+
Mirrors the ``same_seg & causal`` SDPA mask of Megatron-Bridge's dense path.
|
| 159 |
+
|
| 160 |
+
NOTE: this materialises a [B, 1, S, S] bool tensor (~1 GiB at S=32k, B=1).
|
| 161 |
+
A chunked-query SDPA (as in MB's ``modeling_deepseek_ocr_mcore`` cp>1 loop)
|
| 162 |
+
would avoid it and is a follow-up memory optimisation."""
|
| 163 |
+
B, S = position_ids.shape
|
| 164 |
+
seg = segment_ids(position_ids) # [B, S]
|
| 165 |
+
pos = torch.arange(S, device=device) # global positions
|
| 166 |
+
causal = pos[None, :] <= pos[:, None] # [S, S] q >= k
|
| 167 |
+
same_seg = seg[:, :, None] == seg[:, None, :] # [B, S, S]
|
| 168 |
+
return (same_seg & causal[None])[:, None] # [B, 1, S, S] bool
|
pyproject.toml
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=61"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "unlimited-ocr-config-runtime"
|
| 7 |
+
version = "0.1.0"
|
| 8 |
+
requires-python = ">=3.10"
|
| 9 |
+
|
| 10 |
+
[tool.setuptools.packages.find]
|
| 11 |
+
include = ["unlimited_ocr_runtime*"]
|
sparse_gather_e2e_triton.py
ADDED
|
@@ -0,0 +1,407 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""fused Triton gather-attention WITH a differentiable per-block bias.
|
| 2 |
+
|
| 3 |
+
This is the high-performance backend for HiLS-style end-to-end selector training.
|
| 4 |
+
It is a copy of ``sparse_gather_triton.py`` (the production sparse-stage kernel, whose
|
| 5 |
+
union/causal/local-window semantics are already battle-tested) with ONE addition:
|
| 6 |
+
|
| 7 |
+
* a per-(token, kv-block) additive bias ``blk_bias[B, T, M]`` (M = num kv blocks) is
|
| 8 |
+
added to the logits of the tokens attended through the SELECTED (remote) blocks --
|
| 9 |
+
exactly HiLS's inter-chunk gate. The forward adds it; the backward returns
|
| 10 |
+
``d_blk_bias`` so the LM-loss gradient flows back through the bias into whatever
|
| 11 |
+
produced it (the indexer q/k projections). Local-window keys get zero bias, matching
|
| 12 |
+
the eager reference in ``DeepseekV2Indexer.e2e_sparse_attention``.
|
| 13 |
+
|
| 14 |
+
The differentiable log-sum-exp block mass itself is still computed in PyTorch (validated
|
| 15 |
+
by test_e2e_math.py); this kernel only replaces the heavy gather + softmax + matmul and
|
| 16 |
+
their backward, and threads the bias gradient through.
|
| 17 |
+
|
| 18 |
+
Semantics (must match the eager e2e reference):
|
| 19 |
+
logit(query t, key j in selected block b) = scale * q_t . k_j + blk_bias[t, b]
|
| 20 |
+
for j causal and j < local_start(t)
|
| 21 |
+
logit(query t, key j in local window) = scale * q_t . k_j (bias 0)
|
| 22 |
+
Single index head (block indices shared across kv heads), single contiguous causal
|
| 23 |
+
sequence, q_len == k_len (training prefill). The wrapper returns None on unsupported
|
| 24 |
+
shapes so the caller falls back to the eager torch path.
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
import torch
|
| 28 |
+
import triton
|
| 29 |
+
import triton.language as tl
|
| 30 |
+
|
| 31 |
+
# Force transformers' dynamic-module loader to copy this file (see the twin sentinel
|
| 32 |
+
# in sparse_gather_triton.py). modeling_unlimitedocr.py imports this name at top level.
|
| 33 |
+
SPARSE_E2E_SENTINEL = 1
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
@triton.jit(do_not_specialize=['T'])
|
| 37 |
+
def _e2e_attn_fwd_kernel(
|
| 38 |
+
q, k, v, o, lse, scale, block_indices, blk_bias,
|
| 39 |
+
T,
|
| 40 |
+
H: tl.constexpr, HQ: tl.constexpr, G: tl.constexpr, GP: tl.constexpr,
|
| 41 |
+
K: tl.constexpr, V: tl.constexpr, S: tl.constexpr, M: tl.constexpr,
|
| 42 |
+
BS: tl.constexpr, BK: tl.constexpr, BV: tl.constexpr,
|
| 43 |
+
LW: tl.constexpr, NLB: tl.constexpr,
|
| 44 |
+
):
|
| 45 |
+
i_t, i_v, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
| 46 |
+
i_b, i_h = i_bh // H, i_bh % H
|
| 47 |
+
bos = i_b * T
|
| 48 |
+
|
| 49 |
+
k += (bos * H + i_h) * K
|
| 50 |
+
v += (bos * H + i_h) * V
|
| 51 |
+
block_indices += (bos + i_t) * S
|
| 52 |
+
blk_bias += (bos + i_t) * M
|
| 53 |
+
|
| 54 |
+
o_g = tl.arange(0, GP)
|
| 55 |
+
o_k = tl.arange(0, BK)
|
| 56 |
+
o_v = tl.arange(0, BV)
|
| 57 |
+
m_g = o_g < G
|
| 58 |
+
m_k = o_k < K
|
| 59 |
+
m_v = (i_v * BV + o_v) < V
|
| 60 |
+
|
| 61 |
+
p_q = q + (bos + i_t) * HQ * K + (i_h * G + o_g)[:, None] * K + o_k[None, :]
|
| 62 |
+
b_q = tl.load(p_q, mask=m_g[:, None] & m_k[None, :], other=0.)
|
| 63 |
+
b_q = (b_q * scale).to(b_q.dtype)
|
| 64 |
+
|
| 65 |
+
b_o = tl.zeros([GP, BV], dtype=tl.float32)
|
| 66 |
+
b_m = tl.full([GP], float('-inf'), dtype=tl.float32)
|
| 67 |
+
b_acc = tl.zeros([GP], dtype=tl.float32)
|
| 68 |
+
ls = i_t - LW + 1
|
| 69 |
+
|
| 70 |
+
# --- remote: selected non-local blocks (keys < local_start), WITH per-block bias ---
|
| 71 |
+
for i in range(S):
|
| 72 |
+
bid = tl.load(block_indices + i).to(tl.int32)
|
| 73 |
+
i_s = bid * BS
|
| 74 |
+
if i_s <= i_t and i_s >= 0:
|
| 75 |
+
b_bias = tl.load(blk_bias + bid).to(tl.float32)
|
| 76 |
+
p_k = tl.make_block_ptr(k, (K, T), (1, H * K), (0, i_s), (BK, BS), (0, 1))
|
| 77 |
+
p_v = tl.make_block_ptr(v, (T, V), (H * V, 1), (i_s, i_v * BV), (BS, BV), (1, 0))
|
| 78 |
+
b_k = tl.load(p_k, boundary_check=(0, 1))
|
| 79 |
+
b_v = tl.load(p_v, boundary_check=(0, 1))
|
| 80 |
+
b_s = tl.dot(b_q, b_k)
|
| 81 |
+
key = i_s + tl.arange(0, BS)
|
| 82 |
+
b_s = tl.where(((i_t >= key) & (key < ls))[None, :], b_s + b_bias, float('-inf'))
|
| 83 |
+
# safe online softmax: a fully-masked tile keeps b_m at -inf without NaN
|
| 84 |
+
# (b_m_safe==0 there, so alpha=exp(-inf)=0 and b_p=exp(-inf)=0).
|
| 85 |
+
b_m_new = tl.maximum(b_m, tl.max(b_s, 1))
|
| 86 |
+
b_m_safe = tl.where(b_m_new == float('-inf'), 0.0, b_m_new)
|
| 87 |
+
b_r = tl.exp(b_m - b_m_safe)
|
| 88 |
+
b_p = tl.exp(b_s - b_m_safe[:, None])
|
| 89 |
+
b_acc = b_acc * b_r + tl.sum(b_p, 1)
|
| 90 |
+
b_o = b_o * b_r[:, None] + tl.dot(b_p.to(b_q.dtype), b_v)
|
| 91 |
+
b_m = b_m_new
|
| 92 |
+
|
| 93 |
+
# --- local window blocks (keys >= local_start), NO bias ---
|
| 94 |
+
q_blk = i_t // BS
|
| 95 |
+
for lb in range(NLB):
|
| 96 |
+
i_s = (q_blk - lb) * BS
|
| 97 |
+
if i_s >= 0:
|
| 98 |
+
p_k = tl.make_block_ptr(k, (K, T), (1, H * K), (0, i_s), (BK, BS), (0, 1))
|
| 99 |
+
p_v = tl.make_block_ptr(v, (T, V), (H * V, 1), (i_s, i_v * BV), (BS, BV), (1, 0))
|
| 100 |
+
b_k = tl.load(p_k, boundary_check=(0, 1))
|
| 101 |
+
b_v = tl.load(p_v, boundary_check=(0, 1))
|
| 102 |
+
b_s = tl.dot(b_q, b_k)
|
| 103 |
+
key = i_s + tl.arange(0, BS)
|
| 104 |
+
b_s = tl.where(((i_t >= key) & (key >= ls))[None, :], b_s, float('-inf'))
|
| 105 |
+
b_m_new = tl.maximum(b_m, tl.max(b_s, 1))
|
| 106 |
+
b_m_safe = tl.where(b_m_new == float('-inf'), 0.0, b_m_new)
|
| 107 |
+
b_r = tl.exp(b_m - b_m_safe)
|
| 108 |
+
b_p = tl.exp(b_s - b_m_safe[:, None])
|
| 109 |
+
b_acc = b_acc * b_r + tl.sum(b_p, 1)
|
| 110 |
+
b_o = b_o * b_r[:, None] + tl.dot(b_p.to(b_q.dtype), b_v)
|
| 111 |
+
b_m = b_m_new
|
| 112 |
+
|
| 113 |
+
b_safe = tl.where(b_acc > 0, b_acc, 1.0)
|
| 114 |
+
b_o = b_o / b_safe[:, None]
|
| 115 |
+
b_lse = b_m + tl.log(b_safe)
|
| 116 |
+
|
| 117 |
+
p_o = o + (bos + i_t) * HQ * V + (i_h * G + o_g)[:, None] * V + (i_v * BV + o_v)[None, :]
|
| 118 |
+
tl.store(p_o, b_o.to(q.dtype.element_ty), mask=m_g[:, None] & m_v[None, :])
|
| 119 |
+
if i_v == 0:
|
| 120 |
+
p_lse = lse + (bos + i_t) * HQ + i_h * G + o_g
|
| 121 |
+
tl.store(p_lse, b_lse, mask=m_g)
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
@triton.jit(do_not_specialize=['T'])
|
| 125 |
+
def _e2e_attn_bwd_dq_kernel(
|
| 126 |
+
q, k, v, lse, delta, do, dq, scale, block_indices, blk_bias, d_bias,
|
| 127 |
+
T,
|
| 128 |
+
H: tl.constexpr, HQ: tl.constexpr, G: tl.constexpr, GP: tl.constexpr,
|
| 129 |
+
K: tl.constexpr, V: tl.constexpr, S: tl.constexpr, M: tl.constexpr,
|
| 130 |
+
BS: tl.constexpr, BK: tl.constexpr, BV: tl.constexpr,
|
| 131 |
+
LW: tl.constexpr, NLB: tl.constexpr,
|
| 132 |
+
):
|
| 133 |
+
i_t, i_v, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
| 134 |
+
i_b, i_h = i_bh // H, i_bh % H
|
| 135 |
+
bos = i_b * T
|
| 136 |
+
|
| 137 |
+
k += (bos * H + i_h) * K
|
| 138 |
+
v += (bos * H + i_h) * V
|
| 139 |
+
block_indices += (bos + i_t) * S
|
| 140 |
+
blk_bias += (bos + i_t) * M
|
| 141 |
+
d_bias += (bos + i_t) * M
|
| 142 |
+
|
| 143 |
+
o_g = tl.arange(0, GP)
|
| 144 |
+
o_k = tl.arange(0, BK)
|
| 145 |
+
o_v = tl.arange(0, BV)
|
| 146 |
+
m_g = o_g < G
|
| 147 |
+
m_k = o_k < K
|
| 148 |
+
m_v = (i_v * BV + o_v) < V
|
| 149 |
+
|
| 150 |
+
p_q = q + (bos + i_t) * HQ * K + (i_h * G + o_g)[:, None] * K + o_k[None, :]
|
| 151 |
+
b_q = tl.load(p_q, mask=m_g[:, None] & m_k[None, :], other=0.)
|
| 152 |
+
b_q = (b_q * scale).to(b_q.dtype)
|
| 153 |
+
|
| 154 |
+
p_do = do + (bos + i_t) * HQ * V + (i_h * G + o_g)[:, None] * V + (i_v * BV + o_v)[None, :]
|
| 155 |
+
b_do = tl.load(p_do, mask=m_g[:, None] & m_v[None, :], other=0.)
|
| 156 |
+
p_lse = lse + (bos + i_t) * HQ + i_h * G + o_g
|
| 157 |
+
b_lse = tl.load(p_lse, mask=m_g, other=float('inf'))
|
| 158 |
+
p_delta = delta + (bos + i_t) * HQ + i_h * G + o_g
|
| 159 |
+
b_delta = tl.load(p_delta, mask=m_g, other=0.)
|
| 160 |
+
|
| 161 |
+
b_dq = tl.zeros([GP, BK], dtype=tl.float32)
|
| 162 |
+
ls = i_t - LW + 1
|
| 163 |
+
|
| 164 |
+
# --- remote: selected non-local blocks (bias added; accumulate d_bias) ---
|
| 165 |
+
for i in range(S):
|
| 166 |
+
bid = tl.load(block_indices + i).to(tl.int32)
|
| 167 |
+
i_s = bid * BS
|
| 168 |
+
if i_s <= i_t and i_s >= 0:
|
| 169 |
+
b_bias = tl.load(blk_bias + bid).to(tl.float32)
|
| 170 |
+
p_k = tl.make_block_ptr(k, (K, T), (1, H * K), (0, i_s), (BK, BS), (0, 1))
|
| 171 |
+
p_v = tl.make_block_ptr(v, (V, T), (1, H * V), (i_v * BV, i_s), (BV, BS), (0, 1))
|
| 172 |
+
b_k = tl.load(p_k, boundary_check=(0, 1))
|
| 173 |
+
b_v = tl.load(p_v, boundary_check=(0, 1))
|
| 174 |
+
b_s = tl.dot(b_q, b_k)
|
| 175 |
+
key = i_s + tl.arange(0, BS)
|
| 176 |
+
b_p = tl.exp((b_s + b_bias) - b_lse[:, None])
|
| 177 |
+
b_p = tl.where(((i_t >= key) & (key < ls))[None, :], b_p, 0.)
|
| 178 |
+
b_dp = tl.dot(b_do, b_v)
|
| 179 |
+
b_ds = b_p * (b_dp.to(tl.float32) - b_delta[:, None])
|
| 180 |
+
b_dq += tl.dot(b_ds.to(b_k.dtype), tl.trans(b_k))
|
| 181 |
+
# d_bias[t, bid] += sum over this kv-head-group's query rows & keys of ds.
|
| 182 |
+
# Atomics accumulate the remaining kv heads (other programs); padded head
|
| 183 |
+
# rows have lse=+inf -> b_p=0, non-kept keys -> b_p=0, so they add nothing.
|
| 184 |
+
if i_v == 0:
|
| 185 |
+
tl.atomic_add(d_bias + bid, tl.sum(b_ds))
|
| 186 |
+
|
| 187 |
+
# --- local window blocks (no bias, no d_bias) ---
|
| 188 |
+
q_blk = i_t // BS
|
| 189 |
+
for lb in range(NLB):
|
| 190 |
+
i_s = (q_blk - lb) * BS
|
| 191 |
+
if i_s >= 0:
|
| 192 |
+
p_k = tl.make_block_ptr(k, (K, T), (1, H * K), (0, i_s), (BK, BS), (0, 1))
|
| 193 |
+
p_v = tl.make_block_ptr(v, (V, T), (1, H * V), (i_v * BV, i_s), (BV, BS), (0, 1))
|
| 194 |
+
b_k = tl.load(p_k, boundary_check=(0, 1))
|
| 195 |
+
b_v = tl.load(p_v, boundary_check=(0, 1))
|
| 196 |
+
b_s = tl.dot(b_q, b_k)
|
| 197 |
+
key = i_s + tl.arange(0, BS)
|
| 198 |
+
b_p = tl.exp(b_s - b_lse[:, None])
|
| 199 |
+
b_p = tl.where(((i_t >= key) & (key >= ls))[None, :], b_p, 0.)
|
| 200 |
+
b_dp = tl.dot(b_do, b_v)
|
| 201 |
+
b_ds = b_p * (b_dp.to(tl.float32) - b_delta[:, None])
|
| 202 |
+
b_dq += tl.dot(b_ds.to(b_k.dtype), tl.trans(b_k))
|
| 203 |
+
b_dq *= scale
|
| 204 |
+
|
| 205 |
+
p_dq = dq + (bos + i_t) * HQ * K + (i_h * G + o_g)[:, None] * K + o_k[None, :]
|
| 206 |
+
tl.store(p_dq, b_dq.to(q.dtype.element_ty), mask=m_g[:, None] & m_k[None, :])
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
@triton.jit(do_not_specialize=['T'])
|
| 210 |
+
def _e2e_attn_bwd_dkv_kernel(
|
| 211 |
+
q, k, v, lse, delta, do, dk, dv, block_mask, blk_bias, scale,
|
| 212 |
+
T,
|
| 213 |
+
H: tl.constexpr, HQ: tl.constexpr, G: tl.constexpr, GP: tl.constexpr,
|
| 214 |
+
K: tl.constexpr, V: tl.constexpr, M: tl.constexpr,
|
| 215 |
+
BS: tl.constexpr, BK: tl.constexpr, BV: tl.constexpr,
|
| 216 |
+
LW: tl.constexpr,
|
| 217 |
+
):
|
| 218 |
+
i_v, i_s, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
| 219 |
+
i_b, i_h = i_bh // H, i_bh % H
|
| 220 |
+
bos = i_b * T
|
| 221 |
+
|
| 222 |
+
p_k = tl.make_block_ptr(k + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_s * BS, 0), (BS, BK), (1, 0))
|
| 223 |
+
p_v = tl.make_block_ptr(v + (bos * H + i_h) * V, (T, V), (H * V, 1), (i_s * BS, i_v * BV), (BS, BV), (1, 0))
|
| 224 |
+
p_dk = tl.make_block_ptr(dk + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_s * BS, 0), (BS, BK), (1, 0))
|
| 225 |
+
p_dv = tl.make_block_ptr(dv + (bos * H + i_h) * V, (T, V), (H * V, 1), (i_s * BS, i_v * BV), (BS, BV), (1, 0))
|
| 226 |
+
|
| 227 |
+
o_g = tl.arange(0, GP)
|
| 228 |
+
o_k = tl.arange(0, BK)
|
| 229 |
+
o_v = tl.arange(0, BV)
|
| 230 |
+
m_g = o_g < G
|
| 231 |
+
m_k = o_k < K
|
| 232 |
+
m_v = (i_v * BV + o_v) < V
|
| 233 |
+
|
| 234 |
+
b_k = tl.load(p_k, boundary_check=(0, 1))
|
| 235 |
+
b_dk = tl.zeros([BS, BK], dtype=tl.float32)
|
| 236 |
+
b_v = tl.load(p_v, boundary_check=(0, 1))
|
| 237 |
+
b_dv = tl.zeros([BS, BV], dtype=tl.float32)
|
| 238 |
+
|
| 239 |
+
key = i_s * BS + tl.arange(0, BS) # [BS] key positions of this kv block
|
| 240 |
+
|
| 241 |
+
for i in range(i_s * BS, T):
|
| 242 |
+
b_flag = tl.load(block_mask + (bos + i) * M + i_s)
|
| 243 |
+
_proc = (b_flag != 0) | (i < (i_s + 1) * BS + LW)
|
| 244 |
+
if _proc:
|
| 245 |
+
p_q = q + (bos + i) * HQ * K + (i_h * G + o_g)[:, None] * K + o_k[None, :]
|
| 246 |
+
b_q = tl.load(p_q, mask=m_g[:, None] & m_k[None, :], other=0.)
|
| 247 |
+
b_q = (b_q * scale).to(b_q.dtype)
|
| 248 |
+
|
| 249 |
+
p_do = do + (bos + i) * HQ * V + (i_h * G + o_g)[:, None] * V + (i_v * BV + o_v)[None, :]
|
| 250 |
+
b_do = tl.load(p_do, mask=m_g[:, None] & m_v[None, :], other=0.)
|
| 251 |
+
p_lse = lse + (bos + i) * HQ + i_h * G + o_g
|
| 252 |
+
b_lse = tl.load(p_lse, mask=m_g, other=float('inf'))
|
| 253 |
+
p_delta = delta + (bos + i) * HQ + i_h * G + o_g
|
| 254 |
+
b_delta = tl.load(p_delta, mask=m_g, other=0.)
|
| 255 |
+
|
| 256 |
+
ls = i - LW + 1
|
| 257 |
+
is_local = key > (i - LW) # [BS]
|
| 258 |
+
is_remote = key < ls # [BS]
|
| 259 |
+
# bias applies only to REMOTE keys of a SELECTED block (dense cell is 0 for
|
| 260 |
+
# non-selected blocks) -> matches the fwd remote-loop bias, local keys get 0.
|
| 261 |
+
b_bias = tl.load(blk_bias + (bos + i) * M + i_s).to(tl.float32)
|
| 262 |
+
bias_key = tl.where(is_remote, b_bias, 0.0) # [BS]
|
| 263 |
+
|
| 264 |
+
b_s = tl.dot(b_k, tl.trans(b_q)) # [BS, GP]
|
| 265 |
+
b_p = tl.exp((b_s + bias_key[:, None]) - b_lse[None, :])
|
| 266 |
+
causal = i >= key
|
| 267 |
+
keep = causal & (is_local | (b_flag != 0))
|
| 268 |
+
b_p = tl.where(keep[:, None], b_p, 0.)
|
| 269 |
+
b_dv += tl.dot(b_p.to(b_do.dtype), b_do)
|
| 270 |
+
b_dp = tl.dot(b_v, tl.trans(b_do))
|
| 271 |
+
b_ds = b_p * (b_dp.to(tl.float32) - b_delta[None, :])
|
| 272 |
+
b_dk += tl.dot(b_ds.to(b_q.dtype), b_q)
|
| 273 |
+
|
| 274 |
+
tl.store(p_dk, b_dk.to(q.dtype.element_ty), boundary_check=(0, 1))
|
| 275 |
+
tl.store(p_dv, b_dv.to(q.dtype.element_ty), boundary_check=(0, 1))
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def _meta(q, k, v, sel, block_size):
|
| 279 |
+
B, T, HQ, K = q.shape
|
| 280 |
+
H = k.shape[2]
|
| 281 |
+
V = v.shape[3]
|
| 282 |
+
S = sel.shape[-1]
|
| 283 |
+
G = HQ // H
|
| 284 |
+
GP = max(16, triton.next_power_of_2(G))
|
| 285 |
+
BK = triton.next_power_of_2(K)
|
| 286 |
+
BV = min(128, triton.next_power_of_2(V))
|
| 287 |
+
if BV < V:
|
| 288 |
+
raise NotImplementedError(f"head_dim V={V} > 128 not supported")
|
| 289 |
+
if BK > 256:
|
| 290 |
+
raise NotImplementedError(f"head_dim K={K} > 256 not supported")
|
| 291 |
+
return B, T, HQ, H, K, V, S, G, GP, block_size, BK, BV
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
def _local_meta(block_size, local_window):
|
| 295 |
+
lw = block_size if local_window is None else int(local_window)
|
| 296 |
+
lw = max(1, lw)
|
| 297 |
+
nlb = (lw + block_size - 1) // block_size + 1
|
| 298 |
+
return lw, nlb
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
class _E2EGatherAttnFn(torch.autograd.Function):
|
| 302 |
+
@staticmethod
|
| 303 |
+
def forward(ctx, q, k, v, sel, blk_bias, block_size, scale, local_window):
|
| 304 |
+
B, T, HQ, H, K, V, S, G, GP, BS, BK, BV = _meta(q, k, v, sel, block_size)
|
| 305 |
+
LW, NLB = _local_meta(BS, local_window)
|
| 306 |
+
M = triton.cdiv(T, BS)
|
| 307 |
+
assert blk_bias.shape[-1] == M, f"blk_bias last dim {blk_bias.shape[-1]} != M {M}"
|
| 308 |
+
blk_bias = blk_bias.contiguous().float()
|
| 309 |
+
|
| 310 |
+
o = torch.empty(B, T, HQ, V, dtype=q.dtype, device=q.device)
|
| 311 |
+
lse = torch.empty(B, T, HQ, dtype=torch.float32, device=q.device)
|
| 312 |
+
_e2e_attn_fwd_kernel[(T, 1, B * H)](
|
| 313 |
+
q, k, v, o, lse, scale, sel, blk_bias,
|
| 314 |
+
T=T, H=H, HQ=HQ, G=G, GP=GP, K=K, V=V, S=S, M=M, BS=BS, BK=BK, BV=BV,
|
| 315 |
+
LW=LW, NLB=NLB, num_warps=4,
|
| 316 |
+
)
|
| 317 |
+
ctx.save_for_backward(q, k, v, sel, blk_bias, o, lse)
|
| 318 |
+
ctx.block_size = block_size
|
| 319 |
+
ctx.scale = scale
|
| 320 |
+
ctx.local_window = local_window
|
| 321 |
+
return o
|
| 322 |
+
|
| 323 |
+
@staticmethod
|
| 324 |
+
def backward(ctx, do):
|
| 325 |
+
q, k, v, sel, blk_bias, o, lse = ctx.saved_tensors
|
| 326 |
+
block_size, scale, local_window = ctx.block_size, ctx.scale, ctx.local_window
|
| 327 |
+
B, T, HQ, H, K, V, S, G, GP, BS, BK, BV = _meta(q, k, v, sel, block_size)
|
| 328 |
+
LW, NLB = _local_meta(BS, local_window)
|
| 329 |
+
M = triton.cdiv(T, BS)
|
| 330 |
+
do = do.contiguous()
|
| 331 |
+
delta = (o.float() * do.float()).sum(-1) # [B, T, HQ]
|
| 332 |
+
|
| 333 |
+
dq = torch.empty_like(q)
|
| 334 |
+
d_bias = torch.zeros(B, T, M, dtype=torch.float32, device=q.device)
|
| 335 |
+
_e2e_attn_bwd_dq_kernel[(T, 1, B * H)](
|
| 336 |
+
q, k, v, lse, delta, do, dq, scale, sel, blk_bias, d_bias,
|
| 337 |
+
T=T, H=H, HQ=HQ, G=G, GP=GP, K=K, V=V, S=S, M=M, BS=BS, BK=BK, BV=BV,
|
| 338 |
+
LW=LW, NLB=NLB, num_warps=4,
|
| 339 |
+
)
|
| 340 |
+
|
| 341 |
+
block_mask = torch.zeros(B, T, M + 1, dtype=torch.bool, device=q.device)
|
| 342 |
+
block_mask.scatter_(-1, sel.long().clamp(min=0).masked_fill(sel < 0, M), True)
|
| 343 |
+
block_mask = block_mask[..., :M].contiguous()
|
| 344 |
+
|
| 345 |
+
dk = torch.empty_like(k)
|
| 346 |
+
dv = torch.empty_like(v)
|
| 347 |
+
_e2e_attn_bwd_dkv_kernel[(1, M, B * H)](
|
| 348 |
+
q, k, v, lse, delta, do, dk, dv, block_mask, blk_bias, scale,
|
| 349 |
+
T=T, H=H, HQ=HQ, G=G, GP=GP, K=K, V=V, M=M, BS=BS, BK=BK, BV=BV,
|
| 350 |
+
LW=LW, num_warps=4,
|
| 351 |
+
)
|
| 352 |
+
return dq, dk, dv, None, d_bias, None, None, None
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
def triton_gather_attention_e2e(
|
| 356 |
+
query_states, # [B, HQ, T, D]
|
| 357 |
+
key_states, # [B, Hkv, Sk, D]
|
| 358 |
+
value_states, # [B, Hkv, Sk, D]
|
| 359 |
+
block_indices, # [B, 1, T, S] int, -1 = padding
|
| 360 |
+
blk_bias_slots, # [B, T, S] float, differentiable per-selected-block bias (aligned with block_indices)
|
| 361 |
+
position_ids, # [B, T]
|
| 362 |
+
attention_mask,
|
| 363 |
+
block_size,
|
| 364 |
+
scale,
|
| 365 |
+
local_window=None,
|
| 366 |
+
):
|
| 367 |
+
"""Fused e2e gather-attention entry. Returns [B, HQ, T, D], or None on unsupported
|
| 368 |
+
shape (caller must fall back to the eager torch path). Gradients flow to q/k/v AND
|
| 369 |
+
to blk_bias_slots (-> the indexer projections)."""
|
| 370 |
+
if attention_mask is not None:
|
| 371 |
+
return None
|
| 372 |
+
if block_indices.shape[1] != 1:
|
| 373 |
+
return None
|
| 374 |
+
B, HQ, T, D = query_states.shape
|
| 375 |
+
if key_states.shape[2] != T:
|
| 376 |
+
return None
|
| 377 |
+
if position_ids is None or position_ids.dim() != 2 or position_ids.shape[-1] != T:
|
| 378 |
+
return None
|
| 379 |
+
ar = torch.arange(T, device=position_ids.device)
|
| 380 |
+
if not bool((position_ids == ar).all()):
|
| 381 |
+
return None
|
| 382 |
+
|
| 383 |
+
sel = block_indices[:, 0].contiguous() # [B, T, S]
|
| 384 |
+
bias_slots = blk_bias_slots # [B, T, S] (grad)
|
| 385 |
+
M = triton.cdiv(T, block_size)
|
| 386 |
+
|
| 387 |
+
# dedup repeated block ids within a row (would double-count softmax mass).
|
| 388 |
+
valid = sel >= 0
|
| 389 |
+
sel_sorted, order = sel.sort(dim=-1)
|
| 390 |
+
dup_sorted = torch.zeros_like(valid)
|
| 391 |
+
dup_sorted[..., 1:] = sel_sorted[..., 1:] == sel_sorted[..., :-1]
|
| 392 |
+
dup = torch.zeros_like(valid)
|
| 393 |
+
dup.scatter_(-1, order, dup_sorted)
|
| 394 |
+
sel = sel.masked_fill(dup & valid, -1)
|
| 395 |
+
|
| 396 |
+
# scatter per-slot bias into a dense [B, T, M] tensor keyed by block id (differentiable
|
| 397 |
+
# w.r.t. bias_slots). -1 slots (pad/dedup) route to an overflow column then dropped.
|
| 398 |
+
scatter_idx = sel.long().clamp(min=0).masked_fill(sel < 0, M) # [B, T, S]
|
| 399 |
+
blk_bias_dense = bias_slots.new_zeros(B, T, M + 1).scatter(-1, scatter_idx, bias_slots)
|
| 400 |
+
blk_bias_dense = blk_bias_dense[..., :M].contiguous()
|
| 401 |
+
|
| 402 |
+
sel = sel.to(torch.int32).contiguous()
|
| 403 |
+
q = query_states.transpose(1, 2).contiguous() # [B, T, HQ, D]
|
| 404 |
+
k = key_states.transpose(1, 2).contiguous()
|
| 405 |
+
v = value_states.transpose(1, 2).contiguous()
|
| 406 |
+
o = _E2EGatherAttnFn.apply(q, k, v, sel, blk_bias_dense, block_size, scale, local_window)
|
| 407 |
+
return o.transpose(1, 2) # [B, HQ, T, D]
|
sparse_gather_triton.py
ADDED
|
@@ -0,0 +1,524 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""fused Triton kernels for gather-style block-sparse attention
|
| 2 |
+
(per-token top-k kv blocks, single shared index head).
|
| 3 |
+
|
| 4 |
+
Adapted from flash-linear-attention's NSA kernels (fla/ops/nsa/parallel.py), with
|
| 5 |
+
three modifications for this model:
|
| 6 |
+
1. Small GQA group support: HQ/Hkv = 2 here, but tl.dot needs M >= 16, so the
|
| 7 |
+
head-group tile is padded to GP=16 with masked loads/stores (the kernel is
|
| 8 |
+
memory-bound on K/V block loads, the padded tensor-core lanes are ~free).
|
| 9 |
+
2. Multi index-head support ([B, HI, T, S] int32, -1 = pad). Each main-attn head
|
| 10 |
+
hm uses index head hm // (HQ // HI); equivalently kv head i_h uses index head
|
| 11 |
+
(i_h * G) // (HQ // HI). HI == 1 reduces to the shared-index-head case.
|
| 12 |
+
Requires num_kv_heads % HI == 0 so a kv-head tile maps to one index head.
|
| 13 |
+
3. No varlen / block_counts support (single contiguous causal sequence; the
|
| 14 |
+
Python wrapper falls back to the eager gather path otherwise).
|
| 15 |
+
|
| 16 |
+
Semantics match Qwen3Indexer.gather_sparse_attention's eager path exactly:
|
| 17 |
+
per-token topk blocks, token-level causality inside blocks, duplicated block ids
|
| 18 |
+
collapsed (deduped to -1 before the kernel), rows with an empty candidate set
|
| 19 |
+
output zeros.
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
import torch
|
| 23 |
+
import triton
|
| 24 |
+
import triton.language as tl
|
| 25 |
+
|
| 26 |
+
# exported sentinel so modeling_unlimitedocr.py can do a top-level relative
|
| 27 |
+
# import (``from .sparse_gather_triton import SPARSE_BLOCK_SIZE_SENTINEL``) and
|
| 28 |
+
# thereby force transformers' dynamic-module loader to copy this file to the HF
|
| 29 |
+
# modules cache. The lazy imports inside modeling_deepseekv2 are guarded by env
|
| 30 |
+
# flags + try/except and thus not detected as *direct* relative imports of the
|
| 31 |
+
# entry module.
|
| 32 |
+
SPARSE_BLOCK_SIZE_SENTINEL = 128
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@triton.jit(do_not_specialize=['T'])
|
| 36 |
+
def _gather_attn_fwd_kernel(
|
| 37 |
+
q, k, v, o, lse, scale, block_indices, seg,
|
| 38 |
+
T,
|
| 39 |
+
H: tl.constexpr, HQ: tl.constexpr, G: tl.constexpr, GP: tl.constexpr,
|
| 40 |
+
K: tl.constexpr, V: tl.constexpr, S: tl.constexpr,
|
| 41 |
+
BS: tl.constexpr, BK: tl.constexpr, BV: tl.constexpr,
|
| 42 |
+
LW: tl.constexpr, NLB: tl.constexpr,
|
| 43 |
+
HI: tl.constexpr, IDXG: tl.constexpr,
|
| 44 |
+
):
|
| 45 |
+
i_t, i_v, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
| 46 |
+
i_b, i_h = i_bh // H, i_bh % H
|
| 47 |
+
bos = i_b * T
|
| 48 |
+
# seg_start = global start index of this query's sample segment
|
| 49 |
+
# (0 for non-packed rows -> identical to the original single-sequence path).
|
| 50 |
+
# Restricting attended keys to [seg_start, i_t] gives per-sample isolation.
|
| 51 |
+
seg_start = tl.load(seg + bos + i_t).to(tl.int32)
|
| 52 |
+
|
| 53 |
+
k += (bos * H + i_h) * K
|
| 54 |
+
v += (bos * H + i_h) * V
|
| 55 |
+
# block_indices: [B, HI, T, S]; select this kv-head's index head.
|
| 56 |
+
i_idx = (i_h * G) // IDXG
|
| 57 |
+
block_indices += ((i_b * HI + i_idx) * T + i_t) * S
|
| 58 |
+
|
| 59 |
+
o_g = tl.arange(0, GP)
|
| 60 |
+
o_k = tl.arange(0, BK)
|
| 61 |
+
o_v = tl.arange(0, BV)
|
| 62 |
+
m_g = o_g < G
|
| 63 |
+
m_k = o_k < K
|
| 64 |
+
m_v = (i_v * BV + o_v) < V
|
| 65 |
+
|
| 66 |
+
p_q = q + (bos + i_t) * HQ * K + (i_h * G + o_g)[:, None] * K + o_k[None, :]
|
| 67 |
+
b_q = tl.load(p_q, mask=m_g[:, None] & m_k[None, :], other=0.)
|
| 68 |
+
b_q = (b_q * scale).to(b_q.dtype)
|
| 69 |
+
|
| 70 |
+
b_o = tl.zeros([GP, BV], dtype=tl.float32)
|
| 71 |
+
b_m = tl.full([GP], float('-inf'), dtype=tl.float32)
|
| 72 |
+
b_acc = tl.zeros([GP], dtype=tl.float32)
|
| 73 |
+
# local window is [local_start, i_t] with local_start = i_t - LW + 1,
|
| 74 |
+
# LW = sparse_local_block * BS (was hardcoded to BS).
|
| 75 |
+
# Selected (non-local) blocks contribute only keys < local_start; the local window is
|
| 76 |
+
# attended separately below. The two key sets are disjoint -> single online softmax over
|
| 77 |
+
# their union == build_block_mask's `block_keep | local_keep` (token-level, exact).
|
| 78 |
+
ls = tl.maximum(i_t - LW + 1, seg_start)
|
| 79 |
+
# --- remote: selected non-local blocks (keys strictly below the local window) ---
|
| 80 |
+
for i in range(S):
|
| 81 |
+
i_s = tl.load(block_indices + i).to(tl.int32) * BS
|
| 82 |
+
if i_s <= i_t and i_s >= 0:
|
| 83 |
+
p_k = tl.make_block_ptr(k, (K, T), (1, H * K), (0, i_s), (BK, BS), (0, 1))
|
| 84 |
+
p_v = tl.make_block_ptr(v, (T, V), (H * V, 1), (i_s, i_v * BV), (BS, BV), (1, 0))
|
| 85 |
+
b_k = tl.load(p_k, boundary_check=(0, 1))
|
| 86 |
+
b_v = tl.load(p_v, boundary_check=(0, 1))
|
| 87 |
+
b_s = tl.dot(b_q, b_k)
|
| 88 |
+
key = i_s + tl.arange(0, BS)
|
| 89 |
+
b_s = tl.where(((i_t >= key) & (key < ls) & (key >= seg_start))[None, :], b_s, float('-inf'))
|
| 90 |
+
b_m, b_mp = tl.maximum(b_m, tl.max(b_s, 1)), b_m
|
| 91 |
+
b_r = tl.exp(b_mp - b_m)
|
| 92 |
+
b_p = tl.exp(b_s - b_m[:, None])
|
| 93 |
+
b_acc = b_acc * b_r + tl.sum(b_p, 1)
|
| 94 |
+
b_o = b_o * b_r[:, None] + tl.dot(b_p.to(b_q.dtype), b_v)
|
| 95 |
+
# --- local: the (<=NLB) physical blocks covering [local_start, i_t] (keys >= local_start) ---
|
| 96 |
+
q_blk = i_t // BS
|
| 97 |
+
for lb in range(NLB):
|
| 98 |
+
i_s = (q_blk - lb) * BS
|
| 99 |
+
if i_s >= 0:
|
| 100 |
+
p_k = tl.make_block_ptr(k, (K, T), (1, H * K), (0, i_s), (BK, BS), (0, 1))
|
| 101 |
+
p_v = tl.make_block_ptr(v, (T, V), (H * V, 1), (i_s, i_v * BV), (BS, BV), (1, 0))
|
| 102 |
+
b_k = tl.load(p_k, boundary_check=(0, 1))
|
| 103 |
+
b_v = tl.load(p_v, boundary_check=(0, 1))
|
| 104 |
+
b_s = tl.dot(b_q, b_k)
|
| 105 |
+
key = i_s + tl.arange(0, BS)
|
| 106 |
+
b_s = tl.where(((i_t >= key) & (key >= ls))[None, :], b_s, float('-inf'))
|
| 107 |
+
b_m, b_mp = tl.maximum(b_m, tl.max(b_s, 1)), b_m
|
| 108 |
+
b_r = tl.exp(b_mp - b_m)
|
| 109 |
+
b_p = tl.exp(b_s - b_m[:, None])
|
| 110 |
+
b_acc = b_acc * b_r + tl.sum(b_p, 1)
|
| 111 |
+
b_o = b_o * b_r[:, None] + tl.dot(b_p.to(b_q.dtype), b_v)
|
| 112 |
+
|
| 113 |
+
# rows with an empty candidate set: b_acc == 0 -> output zeros (not NaN)
|
| 114 |
+
b_safe = tl.where(b_acc > 0, b_acc, 1.0)
|
| 115 |
+
b_o = b_o / b_safe[:, None]
|
| 116 |
+
b_lse = b_m + tl.log(b_safe)
|
| 117 |
+
|
| 118 |
+
p_o = o + (bos + i_t) * HQ * V + (i_h * G + o_g)[:, None] * V + (i_v * BV + o_v)[None, :]
|
| 119 |
+
tl.store(p_o, b_o.to(q.dtype.element_ty), mask=m_g[:, None] & m_v[None, :])
|
| 120 |
+
if i_v == 0:
|
| 121 |
+
p_lse = lse + (bos + i_t) * HQ + i_h * G + o_g
|
| 122 |
+
tl.store(p_lse, b_lse, mask=m_g)
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
@triton.jit(do_not_specialize=['T'])
|
| 126 |
+
def _gather_attn_bwd_dq_kernel(
|
| 127 |
+
q, k, v, lse, delta, do, dq, scale, block_indices, seg,
|
| 128 |
+
T,
|
| 129 |
+
H: tl.constexpr, HQ: tl.constexpr, G: tl.constexpr, GP: tl.constexpr,
|
| 130 |
+
K: tl.constexpr, V: tl.constexpr, S: tl.constexpr,
|
| 131 |
+
BS: tl.constexpr, BK: tl.constexpr, BV: tl.constexpr,
|
| 132 |
+
LW: tl.constexpr, NLB: tl.constexpr,
|
| 133 |
+
HI: tl.constexpr, IDXG: tl.constexpr,
|
| 134 |
+
):
|
| 135 |
+
i_t, i_v, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
| 136 |
+
i_b, i_h = i_bh // H, i_bh % H
|
| 137 |
+
bos = i_b * T
|
| 138 |
+
seg_start = tl.load(seg + bos + i_t).to(tl.int32)
|
| 139 |
+
|
| 140 |
+
k += (bos * H + i_h) * K
|
| 141 |
+
v += (bos * H + i_h) * V
|
| 142 |
+
i_idx = (i_h * G) // IDXG
|
| 143 |
+
block_indices += ((i_b * HI + i_idx) * T + i_t) * S
|
| 144 |
+
|
| 145 |
+
o_g = tl.arange(0, GP)
|
| 146 |
+
o_k = tl.arange(0, BK)
|
| 147 |
+
o_v = tl.arange(0, BV)
|
| 148 |
+
m_g = o_g < G
|
| 149 |
+
m_k = o_k < K
|
| 150 |
+
m_v = (i_v * BV + o_v) < V
|
| 151 |
+
|
| 152 |
+
p_q = q + (bos + i_t) * HQ * K + (i_h * G + o_g)[:, None] * K + o_k[None, :]
|
| 153 |
+
b_q = tl.load(p_q, mask=m_g[:, None] & m_k[None, :], other=0.)
|
| 154 |
+
b_q = (b_q * scale).to(b_q.dtype)
|
| 155 |
+
|
| 156 |
+
p_do = do + (bos + i_t) * HQ * V + (i_h * G + o_g)[:, None] * V + (i_v * BV + o_v)[None, :]
|
| 157 |
+
b_do = tl.load(p_do, mask=m_g[:, None] & m_v[None, :], other=0.)
|
| 158 |
+
p_lse = lse + (bos + i_t) * HQ + i_h * G + o_g
|
| 159 |
+
# padded head rows: lse=+inf -> p = exp(s - inf) = 0, contributes nothing
|
| 160 |
+
b_lse = tl.load(p_lse, mask=m_g, other=float('inf'))
|
| 161 |
+
p_delta = delta + (bos + i_t) * HQ + i_h * G + o_g
|
| 162 |
+
b_delta = tl.load(p_delta, mask=m_g, other=0.)
|
| 163 |
+
|
| 164 |
+
b_dq = tl.zeros([GP, BK], dtype=tl.float32)
|
| 165 |
+
ls = tl.maximum(i_t - LW + 1, seg_start)
|
| 166 |
+
# --- remote: selected non-local blocks (keys strictly below the local window) ---
|
| 167 |
+
for i in range(S):
|
| 168 |
+
i_s = tl.load(block_indices + i).to(tl.int32) * BS
|
| 169 |
+
if i_s <= i_t and i_s >= 0:
|
| 170 |
+
p_k = tl.make_block_ptr(k, (K, T), (1, H * K), (0, i_s), (BK, BS), (0, 1))
|
| 171 |
+
p_v = tl.make_block_ptr(v, (V, T), (1, H * V), (i_v * BV, i_s), (BV, BS), (0, 1))
|
| 172 |
+
b_k = tl.load(p_k, boundary_check=(0, 1))
|
| 173 |
+
b_v = tl.load(p_v, boundary_check=(0, 1))
|
| 174 |
+
b_s = tl.dot(b_q, b_k)
|
| 175 |
+
key = i_s + tl.arange(0, BS)
|
| 176 |
+
b_p = tl.exp(b_s - b_lse[:, None])
|
| 177 |
+
b_p = tl.where(((i_t >= key) & (key < ls) & (key >= seg_start))[None, :], b_p, 0.)
|
| 178 |
+
b_dp = tl.dot(b_do, b_v)
|
| 179 |
+
b_ds = b_p * (b_dp.to(tl.float32) - b_delta[:, None])
|
| 180 |
+
b_dq += tl.dot(b_ds.to(b_k.dtype), tl.trans(b_k))
|
| 181 |
+
# --- local: the (<=NLB) physical blocks covering [local_start, i_t] (keys >= local_start) ---
|
| 182 |
+
q_blk = i_t // BS
|
| 183 |
+
for lb in range(NLB):
|
| 184 |
+
i_s = (q_blk - lb) * BS
|
| 185 |
+
if i_s >= 0:
|
| 186 |
+
p_k = tl.make_block_ptr(k, (K, T), (1, H * K), (0, i_s), (BK, BS), (0, 1))
|
| 187 |
+
p_v = tl.make_block_ptr(v, (V, T), (1, H * V), (i_v * BV, i_s), (BV, BS), (0, 1))
|
| 188 |
+
b_k = tl.load(p_k, boundary_check=(0, 1))
|
| 189 |
+
b_v = tl.load(p_v, boundary_check=(0, 1))
|
| 190 |
+
b_s = tl.dot(b_q, b_k)
|
| 191 |
+
key = i_s + tl.arange(0, BS)
|
| 192 |
+
b_p = tl.exp(b_s - b_lse[:, None])
|
| 193 |
+
b_p = tl.where(((i_t >= key) & (key >= ls))[None, :], b_p, 0.)
|
| 194 |
+
b_dp = tl.dot(b_do, b_v)
|
| 195 |
+
b_ds = b_p * (b_dp.to(tl.float32) - b_delta[:, None])
|
| 196 |
+
b_dq += tl.dot(b_ds.to(b_k.dtype), tl.trans(b_k))
|
| 197 |
+
b_dq *= scale
|
| 198 |
+
|
| 199 |
+
p_dq = dq + (bos + i_t) * HQ * K + (i_h * G + o_g)[:, None] * K + o_k[None, :]
|
| 200 |
+
tl.store(p_dq, b_dq.to(q.dtype.element_ty), mask=m_g[:, None] & m_k[None, :])
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
@triton.jit(do_not_specialize=['T'])
|
| 204 |
+
def _gather_attn_bwd_dkv_kernel(
|
| 205 |
+
q, k, v, lse, delta, do, dk, dv, block_mask, seg, scale,
|
| 206 |
+
T,
|
| 207 |
+
H: tl.constexpr, HQ: tl.constexpr, G: tl.constexpr, GP: tl.constexpr,
|
| 208 |
+
K: tl.constexpr, V: tl.constexpr, M: tl.constexpr,
|
| 209 |
+
BS: tl.constexpr, BK: tl.constexpr, BV: tl.constexpr,
|
| 210 |
+
LW: tl.constexpr,
|
| 211 |
+
HI: tl.constexpr, IDXG: tl.constexpr,
|
| 212 |
+
):
|
| 213 |
+
i_v, i_s, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
| 214 |
+
i_b, i_h = i_bh // H, i_bh % H
|
| 215 |
+
bos = i_b * T
|
| 216 |
+
i_idx = (i_h * G) // IDXG
|
| 217 |
+
|
| 218 |
+
p_k = tl.make_block_ptr(k + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_s * BS, 0), (BS, BK), (1, 0))
|
| 219 |
+
p_v = tl.make_block_ptr(v + (bos * H + i_h) * V, (T, V), (H * V, 1), (i_s * BS, i_v * BV), (BS, BV), (1, 0))
|
| 220 |
+
p_dk = tl.make_block_ptr(dk + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_s * BS, 0), (BS, BK), (1, 0))
|
| 221 |
+
p_dv = tl.make_block_ptr(dv + (bos * H + i_h) * V, (T, V), (H * V, 1), (i_s * BS, i_v * BV), (BS, BV), (1, 0))
|
| 222 |
+
|
| 223 |
+
o_g = tl.arange(0, GP)
|
| 224 |
+
o_k = tl.arange(0, BK)
|
| 225 |
+
o_v = tl.arange(0, BV)
|
| 226 |
+
m_g = o_g < G
|
| 227 |
+
m_k = o_k < K
|
| 228 |
+
m_v = (i_v * BV + o_v) < V
|
| 229 |
+
|
| 230 |
+
# [BS, BK]
|
| 231 |
+
b_k = tl.load(p_k, boundary_check=(0, 1))
|
| 232 |
+
b_dk = tl.zeros([BS, BK], dtype=tl.float32)
|
| 233 |
+
# [BS, BV]
|
| 234 |
+
b_v = tl.load(p_v, boundary_check=(0, 1))
|
| 235 |
+
b_dv = tl.zeros([BS, BV], dtype=tl.float32)
|
| 236 |
+
|
| 237 |
+
for i in range(i_s * BS, T):
|
| 238 |
+
# block_mask: [B, HI, T, M] (selected NON-LOCAL blocks only), per index head.
|
| 239 |
+
b_flag = tl.load(block_mask + ((i_b * HI + i_idx) * T + i) * M + i_s)
|
| 240 |
+
# process query i for this kv block if the block is selected (b_flag)
|
| 241 |
+
# OR the block overlaps query i's local window [i - LW + 1, i]. The block spans
|
| 242 |
+
# keys [i_s*BS, (i_s+1)*BS), so overlap implies i < (i_s+1)*BS + LW (loose bound;
|
| 243 |
+
# the exact per-key filter is `keep` below).
|
| 244 |
+
_proc = (b_flag != 0) | (i < (i_s + 1) * BS + LW)
|
| 245 |
+
if _proc:
|
| 246 |
+
p_q = q + (bos + i) * HQ * K + (i_h * G + o_g)[:, None] * K + o_k[None, :]
|
| 247 |
+
b_q = tl.load(p_q, mask=m_g[:, None] & m_k[None, :], other=0.)
|
| 248 |
+
b_q = (b_q * scale).to(b_q.dtype)
|
| 249 |
+
|
| 250 |
+
p_do = do + (bos + i) * HQ * V + (i_h * G + o_g)[:, None] * V + (i_v * BV + o_v)[None, :]
|
| 251 |
+
b_do = tl.load(p_do, mask=m_g[:, None] & m_v[None, :], other=0.)
|
| 252 |
+
p_lse = lse + (bos + i) * HQ + i_h * G + o_g
|
| 253 |
+
b_lse = tl.load(p_lse, mask=m_g, other=float('inf'))
|
| 254 |
+
p_delta = delta + (bos + i) * HQ + i_h * G + o_g
|
| 255 |
+
b_delta = tl.load(p_delta, mask=m_g, other=0.)
|
| 256 |
+
|
| 257 |
+
# [BS, GP]
|
| 258 |
+
b_s = tl.dot(b_k, tl.trans(b_q))
|
| 259 |
+
b_p = tl.exp(b_s - b_lse[None, :])
|
| 260 |
+
# keep = causal & (local-window | selected-block); matches the fwd union.
|
| 261 |
+
# also require the key to be in query i's segment (key >=
|
| 262 |
+
# seg_start_i) so a boundary-straddling / cross-sample block is trimmed.
|
| 263 |
+
seg_i = tl.load(seg + bos + i).to(tl.int32)
|
| 264 |
+
key = i_s * BS + tl.arange(0, BS) # [BS]
|
| 265 |
+
causal = i >= key # [BS]
|
| 266 |
+
in_seg = key >= seg_i # [BS]
|
| 267 |
+
is_local = key > (i - LW) # [BS]
|
| 268 |
+
keep = causal & in_seg & (is_local | (b_flag != 0))
|
| 269 |
+
b_p = tl.where(keep[:, None], b_p, 0.)
|
| 270 |
+
# [BS, GP] @ [GP, BV] -> [BS, BV]
|
| 271 |
+
b_dv += tl.dot(b_p.to(b_do.dtype), b_do)
|
| 272 |
+
# [BS, BV] @ [BV, GP] -> [BS, GP]
|
| 273 |
+
b_dp = tl.dot(b_v, tl.trans(b_do))
|
| 274 |
+
b_ds = b_p * (b_dp.to(tl.float32) - b_delta[None, :])
|
| 275 |
+
# [BS, GP] @ [GP, BK] -> [BS, BK]
|
| 276 |
+
b_dk += tl.dot(b_ds.to(b_q.dtype), b_q)
|
| 277 |
+
|
| 278 |
+
tl.store(p_dk, b_dk.to(q.dtype.element_ty), boundary_check=(0, 1))
|
| 279 |
+
tl.store(p_dv, b_dv.to(q.dtype.element_ty), boundary_check=(0, 1))
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
def _kernel_meta(q, k, v, sel, block_size):
|
| 283 |
+
B, T, HQ, K = q.shape
|
| 284 |
+
H = k.shape[2]
|
| 285 |
+
V = v.shape[3]
|
| 286 |
+
S = sel.shape[-1]
|
| 287 |
+
G = HQ // H
|
| 288 |
+
GP = max(16, triton.next_power_of_2(G))
|
| 289 |
+
BK = triton.next_power_of_2(K)
|
| 290 |
+
BV = min(128, triton.next_power_of_2(V))
|
| 291 |
+
if BV < V:
|
| 292 |
+
raise NotImplementedError(f"head_dim V={V} > 128 not supported")
|
| 293 |
+
if BK > 256:
|
| 294 |
+
raise NotImplementedError(f"head_dim K={K} > 256 not supported")
|
| 295 |
+
return B, T, HQ, H, K, V, S, G, GP, block_size, BK, BV
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
def _local_meta(block_size, local_window):
|
| 299 |
+
"""local window length in tokens + how many physical blocks can overlap it.
|
| 300 |
+
The window [pos - LW + 1, pos] is not block-aligned, so it spans at most
|
| 301 |
+
ceil(LW / BS) + 1 blocks; with LW a multiple of BS that is LW // BS + 1."""
|
| 302 |
+
lw = block_size if local_window is None else int(local_window)
|
| 303 |
+
lw = max(1, lw)
|
| 304 |
+
nlb = (lw + block_size - 1) // block_size + 1
|
| 305 |
+
return lw, nlb
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
def _gather_attn_fwd(q, k, v, sel, seg, block_size, scale, local_window=None):
|
| 309 |
+
B, T, HQ, H, K, V, S, G, GP, BS, BK, BV = _kernel_meta(q, k, v, sel, block_size)
|
| 310 |
+
HI = sel.shape[1] # number of index heads; sel is [B, HI, T, S]
|
| 311 |
+
IDXG = HQ // HI # main-attn heads per index head
|
| 312 |
+
LW, NLB = _local_meta(BS, local_window)
|
| 313 |
+
o = torch.empty(B, T, HQ, V, dtype=q.dtype, device=q.device)
|
| 314 |
+
lse = torch.empty(B, T, HQ, dtype=torch.float32, device=q.device)
|
| 315 |
+
_gather_attn_fwd_kernel[(T, 1, B * H)](
|
| 316 |
+
q, k, v, o, lse, scale, sel, seg,
|
| 317 |
+
T=T, H=H, HQ=HQ, G=G, GP=GP, K=K, V=V, S=S, BS=BS, BK=BK, BV=BV,
|
| 318 |
+
LW=LW, NLB=NLB, HI=HI, IDXG=IDXG,
|
| 319 |
+
num_warps=4,
|
| 320 |
+
)
|
| 321 |
+
return o, lse
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
def _gather_attn_bwd(q, k, v, o, lse, do, sel, seg, block_size, scale, local_window=None):
|
| 325 |
+
B, T, HQ, H, K, V, S, G, GP, BS, BK, BV = _kernel_meta(q, k, v, sel, block_size)
|
| 326 |
+
HI = sel.shape[1]
|
| 327 |
+
IDXG = HQ // HI
|
| 328 |
+
LW, NLB = _local_meta(BS, local_window)
|
| 329 |
+
delta = (o.float() * do.float()).sum(-1) # [B, T, HQ]
|
| 330 |
+
|
| 331 |
+
dq = torch.empty_like(q)
|
| 332 |
+
_gather_attn_bwd_dq_kernel[(T, 1, B * H)](
|
| 333 |
+
q, k, v, lse, delta, do, dq, scale, sel, seg,
|
| 334 |
+
T=T, H=H, HQ=HQ, G=G, GP=GP, K=K, V=V, S=S, BS=BS, BK=BK, BV=BV,
|
| 335 |
+
LW=LW, NLB=NLB, HI=HI, IDXG=IDXG,
|
| 336 |
+
num_warps=4,
|
| 337 |
+
)
|
| 338 |
+
|
| 339 |
+
M = triton.cdiv(T, BS)
|
| 340 |
+
# per index head: [B, HI, T, M]; -1 slots routed to an overflow column then dropped
|
| 341 |
+
block_mask = torch.zeros(B, HI, T, M + 1, dtype=torch.bool, device=q.device)
|
| 342 |
+
block_mask.scatter_(-1, sel.long().clamp(min=0).masked_fill(sel < 0, M), True)
|
| 343 |
+
block_mask = block_mask[..., :M].contiguous()
|
| 344 |
+
|
| 345 |
+
dk = torch.empty_like(k)
|
| 346 |
+
dv = torch.empty_like(v)
|
| 347 |
+
_gather_attn_bwd_dkv_kernel[(1, M, B * H)](
|
| 348 |
+
q, k, v, lse, delta, do, dk, dv, block_mask, seg, scale,
|
| 349 |
+
T=T, H=H, HQ=HQ, G=G, GP=GP, K=K, V=V, M=M, BS=BS, BK=BK, BV=BV,
|
| 350 |
+
LW=LW, HI=HI, IDXG=IDXG,
|
| 351 |
+
num_warps=4,
|
| 352 |
+
)
|
| 353 |
+
return dq, dk, dv
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
class _TritonGatherAttnFn(torch.autograd.Function):
|
| 357 |
+
"""Note: the decoder layer sits under gradient checkpointing, so the tensors
|
| 358 |
+
saved here (incl. o/lse) only live transiently during that layer's recompute
|
| 359 |
+
+backward window -- not across the whole model."""
|
| 360 |
+
|
| 361 |
+
@staticmethod
|
| 362 |
+
def forward(ctx, q, k, v, sel, seg, block_size, scale, local_window=None):
|
| 363 |
+
o, lse = _gather_attn_fwd(q, k, v, sel, seg, block_size, scale, local_window)
|
| 364 |
+
ctx.save_for_backward(q, k, v, sel, seg, o, lse)
|
| 365 |
+
ctx.block_size = block_size
|
| 366 |
+
ctx.scale = scale
|
| 367 |
+
ctx.local_window = local_window
|
| 368 |
+
return o
|
| 369 |
+
|
| 370 |
+
@staticmethod
|
| 371 |
+
def backward(ctx, do):
|
| 372 |
+
q, k, v, sel, seg, o, lse = ctx.saved_tensors
|
| 373 |
+
dq, dk, dv = _gather_attn_bwd(
|
| 374 |
+
q, k, v, o, lse, do.contiguous(), sel, seg, ctx.block_size, ctx.scale,
|
| 375 |
+
ctx.local_window)
|
| 376 |
+
return dq, dk, dv, None, None, None, None, None
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
def triton_gather_attention(
|
| 380 |
+
query_states, # [B, HQ, T, D]
|
| 381 |
+
key_states, # [B, Hkv, Sk, D]
|
| 382 |
+
value_states, # [B, Hkv, Sk, D]
|
| 383 |
+
block_indices, # [B, H_idx, T, S], -1 = padding
|
| 384 |
+
position_ids, # [B, T]
|
| 385 |
+
attention_mask,
|
| 386 |
+
block_size,
|
| 387 |
+
scale,
|
| 388 |
+
local_window=None,
|
| 389 |
+
):
|
| 390 |
+
"""Fused path entry. Returns [B, HQ, T, D], or None when this input shape is
|
| 391 |
+
unsupported (caller must fall back to the eager gather path):
|
| 392 |
+
padding/attention mask present, prefix kv cache (q_len != k_len), or
|
| 393 |
+
index-head count not dividing the head layout. Supports HI >= 1 index heads
|
| 394 |
+
([B, HI, T, S]).
|
| 395 |
+
|
| 396 |
+
supports sequence packing. A per-token seg_start (global start
|
| 397 |
+
index of each token's sample segment) is derived as ``arange(T) - position_ids``
|
| 398 |
+
(0 for a non-packed plain-arange row) and threaded into the kernels so every
|
| 399 |
+
query only attends within [seg_start, t] -> no cross-sample leakage."""
|
| 400 |
+
if attention_mask is not None:
|
| 401 |
+
return None
|
| 402 |
+
B, HQ, T, D = query_states.shape
|
| 403 |
+
HI = block_indices.shape[1]
|
| 404 |
+
Hkv = key_states.shape[1]
|
| 405 |
+
# each kv-head tile must map to a single index head (HI | Hkv), and index
|
| 406 |
+
# heads must tile the query heads evenly (HI | HQ).
|
| 407 |
+
if HI < 1 or HQ % HI != 0 or Hkv % HI != 0:
|
| 408 |
+
return None
|
| 409 |
+
if key_states.shape[2] != T:
|
| 410 |
+
return None
|
| 411 |
+
if position_ids is None or position_ids.dim() != 2 or position_ids.shape[-1] != T:
|
| 412 |
+
return None
|
| 413 |
+
# seg_start[b, t] = t - position_ids[b, t]. For a plain-arange
|
| 414 |
+
# (non-packed) row this is all zeros -> kernels reduce to the original path.
|
| 415 |
+
ar = torch.arange(T, device=position_ids.device)
|
| 416 |
+
seg = (ar[None, :] - position_ids).to(torch.int32).contiguous()
|
| 417 |
+
|
| 418 |
+
sel = block_indices # [B, HI, T, S]
|
| 419 |
+
# dedup repeated block ids within a row (duplicates would double-count
|
| 420 |
+
# their softmax mass in the online-softmax accumulation)
|
| 421 |
+
valid = sel >= 0
|
| 422 |
+
sel_sorted, order = sel.sort(dim=-1)
|
| 423 |
+
dup_sorted = torch.zeros_like(valid)
|
| 424 |
+
dup_sorted[..., 1:] = sel_sorted[..., 1:] == sel_sorted[..., :-1]
|
| 425 |
+
dup = torch.zeros_like(valid)
|
| 426 |
+
dup.scatter_(-1, order, dup_sorted)
|
| 427 |
+
sel = sel.masked_fill(dup & valid, -1)
|
| 428 |
+
sel = sel.to(torch.int32).contiguous()
|
| 429 |
+
|
| 430 |
+
q = query_states.transpose(1, 2).contiguous() # [B, T, HQ, D]
|
| 431 |
+
k = key_states.transpose(1, 2).contiguous() # [B, T, Hkv, D]
|
| 432 |
+
v = value_states.transpose(1, 2).contiguous()
|
| 433 |
+
o = _TritonGatherAttnFn.apply(q, k, v, sel, seg, block_size, scale, local_window)
|
| 434 |
+
return o.transpose(1, 2) # [B, HQ, T, D]
|
| 435 |
+
|
| 436 |
+
|
| 437 |
+
# ---------------------------------------------------------------------------
|
| 438 |
+
# fused "QK score + block-amax" for the indexer block selection.
|
| 439 |
+
# Produces block_scores[b, c, blk] = max over keys j in block of (q[b,c] . k[b,j])
|
| 440 |
+
# restricted to NON-LOCAL causal keys, i.e. j < local_start[c] = max(0, pos[c]-local_window+1)
|
| 441 |
+
# (this matches the eager path's `token_future | local_token` masking before the block amax).
|
| 442 |
+
# No dense [B, C, Sk] score tensor is materialised. Non-differentiable (selection is
|
| 443 |
+
# argmax/topk); unscaled to match the eager path. Single index head only.
|
| 444 |
+
# ---------------------------------------------------------------------------
|
| 445 |
+
@triton.jit
|
| 446 |
+
def _block_max_score_kernel(
|
| 447 |
+
q_ptr, k_ptr, pos_ptr, seg_ptr, out_ptr,
|
| 448 |
+
C, Sk,
|
| 449 |
+
sqb, sqc, sqd, skb, sks, skd, spb, spc, sgb, sgc, sob, soc, son,
|
| 450 |
+
D: tl.constexpr, BS: tl.constexpr, BC: tl.constexpr, BD: tl.constexpr,
|
| 451 |
+
LW: tl.constexpr,
|
| 452 |
+
):
|
| 453 |
+
i_c = tl.program_id(0) # query tile
|
| 454 |
+
i_blk = tl.program_id(1) # key block
|
| 455 |
+
i_b = tl.program_id(2) # batch
|
| 456 |
+
|
| 457 |
+
off_c = i_c * BC + tl.arange(0, BC) # [BC] query rows
|
| 458 |
+
off_d = tl.arange(0, BD) # [BD] head dim
|
| 459 |
+
m_c = off_c < C
|
| 460 |
+
m_d = off_d < D
|
| 461 |
+
|
| 462 |
+
# [BC, BD] query tile
|
| 463 |
+
b_q = tl.load(q_ptr + i_b * sqb + off_c[:, None] * sqc + off_d[None, :] * sqd,
|
| 464 |
+
mask=m_c[:, None] & m_d[None, :], other=0.0)
|
| 465 |
+
# [BC] query positions (GLOBAL under packing) -> non-local upper bound
|
| 466 |
+
# local_start = max(0, pos-local_window+1); seg_start = per-query segment start
|
| 467 |
+
# (0 non-packed) -> remote keys restricted to [seg_start, local_start).
|
| 468 |
+
b_pos = tl.load(pos_ptr + i_b * spb + off_c * spc, mask=m_c, other=0).to(tl.int32)
|
| 469 |
+
b_seg = tl.load(seg_ptr + i_b * sgb + off_c * sgc, mask=m_c, other=0).to(tl.int32)
|
| 470 |
+
b_ls = tl.maximum(b_pos - (LW - 1), 0)
|
| 471 |
+
|
| 472 |
+
j = i_blk * BS + tl.arange(0, BS) # [BS] key indices for this block
|
| 473 |
+
m_j = j < Sk
|
| 474 |
+
# [BS, BD] key block
|
| 475 |
+
b_k = tl.load(k_ptr + i_b * skb + j[:, None] * sks + off_d[None, :] * skd,
|
| 476 |
+
mask=m_j[:, None] & m_d[None, :], other=0.0)
|
| 477 |
+
|
| 478 |
+
# [BC, BS] raw scores (fp32 accumulate); keep only non-local causal in-range,
|
| 479 |
+
# same-segment keys
|
| 480 |
+
b_s = tl.dot(b_q, tl.trans(b_k))
|
| 481 |
+
valid = (j[None, :] < b_ls[:, None]) & (j[None, :] >= b_seg[:, None]) & m_j[None, :]
|
| 482 |
+
b_s = tl.where(valid, b_s, float('-inf'))
|
| 483 |
+
b_max = tl.max(b_s, axis=1) # [BC] block max per query
|
| 484 |
+
|
| 485 |
+
tl.store(out_ptr + i_b * sob + off_c * soc + i_blk * son, b_max, mask=m_c)
|
| 486 |
+
|
| 487 |
+
|
| 488 |
+
def triton_block_max_score(q, k, pos, seg, block_size, num_key_blocks, local_window=None):
|
| 489 |
+
"""Fused block-level max of q.k over NON-LOCAL causal SAME-SEGMENT keys; no dense
|
| 490 |
+
[B,C,Sk] tensor.
|
| 491 |
+
|
| 492 |
+
q: [B, C, D] (index-branch queries, single index head)
|
| 493 |
+
k: [B, Sk, D] (index-branch keys)
|
| 494 |
+
pos: [B, C] GLOBAL query positions
|
| 495 |
+
seg: [B, C] per-query segment start (0 for non-packed)
|
| 496 |
+
Returns block_scores [B, C, num_key_blocks] float32 (-inf where a block has no
|
| 497 |
+
valid key). Matches the eager `token_future | local_token | ~same_seg` masking."""
|
| 498 |
+
assert q.dim() == 3 and k.dim() == 3, "expects [B, C, D] / [B, Sk, D]"
|
| 499 |
+
B, C, D = q.shape
|
| 500 |
+
Sk = k.shape[1]
|
| 501 |
+
nblk = num_key_blocks
|
| 502 |
+
BD = triton.next_power_of_2(D)
|
| 503 |
+
if BD > 256:
|
| 504 |
+
raise NotImplementedError(f"index dim D={D} > 256 not supported")
|
| 505 |
+
BC = 64
|
| 506 |
+
q = q.contiguous()
|
| 507 |
+
k = k.contiguous()
|
| 508 |
+
pos = pos.contiguous().to(torch.int32)
|
| 509 |
+
seg = seg.contiguous().to(torch.int32)
|
| 510 |
+
out = torch.empty(B, C, nblk, dtype=torch.float32, device=q.device)
|
| 511 |
+
grid = (triton.cdiv(C, BC), nblk, B)
|
| 512 |
+
_block_max_score_kernel[grid](
|
| 513 |
+
q, k, pos, seg, out,
|
| 514 |
+
C, Sk,
|
| 515 |
+
q.stride(0), q.stride(1), q.stride(2),
|
| 516 |
+
k.stride(0), k.stride(1), k.stride(2),
|
| 517 |
+
pos.stride(0), pos.stride(1),
|
| 518 |
+
seg.stride(0), seg.stride(1),
|
| 519 |
+
out.stride(0), out.stride(1), out.stride(2),
|
| 520 |
+
D=D, BS=block_size, BC=BC, BD=BD,
|
| 521 |
+
LW=(block_size if local_window is None else max(1, int(local_window))),
|
| 522 |
+
num_warps=4,
|
| 523 |
+
)
|
| 524 |
+
return out
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"additional_special_tokens": [
|
| 3 |
+
"<|User|>",
|
| 4 |
+
"<|Assistant|>"
|
| 5 |
+
],
|
| 6 |
+
"bos_token": {
|
| 7 |
+
"content": "<|begin▁of▁sentence|>",
|
| 8 |
+
"lstrip": false,
|
| 9 |
+
"normalized": false,
|
| 10 |
+
"rstrip": false,
|
| 11 |
+
"single_word": false
|
| 12 |
+
},
|
| 13 |
+
"eos_token": {
|
| 14 |
+
"content": "<|end▁of▁sentence|>",
|
| 15 |
+
"lstrip": false,
|
| 16 |
+
"normalized": false,
|
| 17 |
+
"rstrip": false,
|
| 18 |
+
"single_word": false
|
| 19 |
+
},
|
| 20 |
+
"pad_token": {
|
| 21 |
+
"content": "<|▁pad▁|>",
|
| 22 |
+
"lstrip": false,
|
| 23 |
+
"normalized": false,
|
| 24 |
+
"rstrip": false,
|
| 25 |
+
"single_word": false
|
| 26 |
+
}
|
| 27 |
+
}
|
tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_config.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
unlimited_ocr_config_runtime.egg-info/PKG-INFO
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Metadata-Version: 2.4
|
| 2 |
+
Name: unlimited-ocr-config-runtime
|
| 3 |
+
Version: 0.1.0
|
| 4 |
+
Requires-Python: >=3.10
|
unlimited_ocr_config_runtime.egg-info/SOURCES.txt
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pyproject.toml
|
| 2 |
+
unlimited_ocr_config_runtime.egg-info/PKG-INFO
|
| 3 |
+
unlimited_ocr_config_runtime.egg-info/SOURCES.txt
|
| 4 |
+
unlimited_ocr_config_runtime.egg-info/dependency_links.txt
|
| 5 |
+
unlimited_ocr_config_runtime.egg-info/top_level.txt
|
| 6 |
+
unlimited_ocr_runtime/__init__.py
|
| 7 |
+
unlimited_ocr_runtime/model.py
|
unlimited_ocr_config_runtime.egg-info/dependency_links.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
|
unlimited_ocr_config_runtime.egg-info/top_level.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
unlimited_ocr_runtime
|
unlimited_ocr_runtime/__init__.py
ADDED
|
File without changes
|
unlimited_ocr_runtime/model.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from copy import deepcopy
|
| 2 |
+
from functools import partial
|
| 3 |
+
from unittest.mock import patch
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
from torch import nn
|
| 7 |
+
from sglang.srt.models import deepseek_ocr as vision
|
| 8 |
+
from sglang.srt.models import unlimited_ocr as base
|
| 9 |
+
from sglang.srt.models import unlimited_ocr_v2 as dash
|
| 10 |
+
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
|
| 11 |
+
from sglang.srt.models.unlimited_ocr_v2 import UnlimitedOCRV2ForCausalLM as UpstreamModel
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class ConfigAttention(dash.UnlimitedOCRV2Attention):
|
| 15 |
+
def __init__(self, config, layer_id, quant_config=None, prefix=""):
|
| 16 |
+
# Use the pinned constructor and forward, with a separate residual width.
|
| 17 |
+
attention_config = deepcopy(config)
|
| 18 |
+
attention_config.hidden_size = config.num_attention_heads * config.head_dim
|
| 19 |
+
super().__init__(attention_config, layer_id, quant_config, prefix)
|
| 20 |
+
self.hidden_size = config.hidden_size
|
| 21 |
+
outputs = {
|
| 22 |
+
"q_proj": config.num_attention_heads * self.head_dim,
|
| 23 |
+
"k_proj": config.num_key_value_heads * self.head_dim,
|
| 24 |
+
"v_proj": config.num_key_value_heads * self.head_dim,
|
| 25 |
+
"index_q_proj": self.idx_head_dim,
|
| 26 |
+
"index_k_proj": self.idx_head_dim,
|
| 27 |
+
}
|
| 28 |
+
for name, output_size in outputs.items():
|
| 29 |
+
setattr(self, name, ColumnParallelLinear(config.hidden_size, output_size,
|
| 30 |
+
bias=False, gather_output=name.startswith("index_"), quant_config=quant_config,
|
| 31 |
+
prefix=f"{prefix}.{name}" if prefix else name))
|
| 32 |
+
self.o_proj = RowParallelLinear(config.num_attention_heads * self.head_dim,
|
| 33 |
+
config.hidden_size, bias=False, quant_config=quant_config,
|
| 34 |
+
prefix=f"{prefix}.o_proj" if prefix else "o_proj")
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class UnlimitedOCRV2ForCausalLM(UpstreamModel):
|
| 38 |
+
def __init__(self, *, config, quant_config=None, prefix=""):
|
| 39 |
+
specs = config.vision_config.width
|
| 40 |
+
|
| 41 |
+
def sam():
|
| 42 |
+
spec = specs["sam_vit_b"]
|
| 43 |
+
encoder = vision.ImageEncoderViT(embed_dim=spec["width"], depth=spec["layers"],
|
| 44 |
+
num_heads=spec["heads"], global_attn_indexes=spec["global_attn_indexes"],
|
| 45 |
+
out_chans=spec["neck_channels"], window_size=14, use_rel_pos=True,
|
| 46 |
+
norm_layer=partial(nn.LayerNorm, eps=1e-6))
|
| 47 |
+
c1, c2 = spec["downsample_channels"]
|
| 48 |
+
encoder.net_2 = nn.Conv2d(spec["neck_channels"], c1, 3, stride=2, padding=1, bias=False)
|
| 49 |
+
encoder.net_3 = nn.Conv2d(c1, c2, 3, stride=2, padding=1, bias=False)
|
| 50 |
+
return encoder
|
| 51 |
+
|
| 52 |
+
def clip():
|
| 53 |
+
spec = specs["clip-l-14-224"]
|
| 54 |
+
cfg = deepcopy(vision.vit_model_cfg)
|
| 55 |
+
cfg.update(num_layers=spec["layers"], hidden_size=spec["width"], num_heads=spec["heads"],
|
| 56 |
+
num_attention_heads=spec["heads"], ffn_hidden_size=spec["width"] * 4)
|
| 57 |
+
return vision.VitModel(cfg)
|
| 58 |
+
|
| 59 |
+
with (patch.object(base, "build_sam_vit_b", sam),
|
| 60 |
+
patch.object(base, "build_clip_l", clip),
|
| 61 |
+
patch.object(dash, "UnlimitedOCRV2Attention", ConfigAttention)):
|
| 62 |
+
super().__init__(config=config, quant_config=quant_config, prefix=prefix)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
EntryClass = UnlimitedOCRV2ForCausalLM
|