Upload DivCenterer v5 1.1M model

Safetensors weights, custom config and tokenizer, implementation, and model card. Greedy exact CSS: 980/980 across the synthetic evaluation suite.

README.md

@@ -0,0 +1,53 @@
+---
+tags:
+- text-generation
+- css
+- synthetic-data
+- custom-model
+---
+
+# DivCenterer
+
+A deliberately tiny, single-task decoder-only Transformer that generates CSS to center one div with either CSS Grid or Flexbox.
+
+## Model
+
+- 1,107,216 parameters; 4 decoder layers, width 144, 4 attention heads, SwiGLU width 384.
+- Trained from scratch with BF16 autocast and PyTorch SDPA on an AMD Radeon RX 9070 XT.
+- Custom 768-token constrained byte-BPE tokenizer; this is not a Transformers `AutoModel` checkpoint.
+- Files: `model.safetensors` (weights), `config.json`, `tokenizer.json`, `model.py`, and `tokenizer.py`.
+
+Load the weights with the included implementation:
+
+```python
+import json
+import torch
+from safetensors.torch import load_file
+from model import DivCentererLM, ModelConfig
+
+with open("config.json", encoding="utf-8") as stream:
+    config = ModelConfig(**json.load(stream))
+model = DivCentererLM(config)
+model.load_state_dict(load_file("model.safetensors"))
+model.eval()
+```
+
+## Data and evaluation
+
+The model was trained on a deterministic synthetic dataset. The prompts and HTML are generated from project-specific templates and word lists; no downloaded corpus or pretrained model weights were used. The model and dataset were developed with AI assistance.
+
+Greedy exact-CSS results on the full synthetic evaluation suite:
+
+| Split | Exact |
+| --- | ---: |
+| Validation | 410/410 |
+| Held-out pairs | 410/410 |
+| Unseen values | 64/64 |
+| Held-out phrasing | 96/96 |
+| **Total** | **980/980** |
+
+These scores measure exact matches against the generated dataset's canonical CSS answers.
+
+## License
+
+No reuse license is specified in this repository.

config.json

@@ -0,0 +1,11 @@
+{
+  "vocab_size": 768,
+  "block_size": 256,
+  "n_layer": 4,
+  "n_head": 4,
+  "n_embd": 144,
+  "ffn_dim": 384,
+  "rope_theta": 10000.0,
+  "dropout": 0.0,
+  "tie_embeddings": true
+}

model.py

@@ -0,0 +1,168 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+import math
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+
+@dataclass
+class ModelConfig:
+    vocab_size: int
+    block_size: int = 256
+    n_layer: int = 3
+    n_head: int = 4
+    n_embd: int = 96
+    ffn_dim: int = 256
+    rope_theta: float = 10000.0
+    dropout: float = 0.0
+    tie_embeddings: bool = True
+
+    def validate(self) -> None:
+        if self.n_embd % self.n_head:
+            raise ValueError("n_embd must be divisible by n_head")
+        if (self.n_embd // self.n_head) % 2:
+            raise ValueError("RoPE requires an even attention head dimension")
+        if self.block_size < 2:
+            raise ValueError("block_size must be at least 2")
+
+
+class RMSNorm(nn.Module):
+    def __init__(self, width: int, eps: float = 1e-6):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(width))
+        self.eps = eps
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        source_dtype = x.dtype
+        normalized = x.float() * torch.rsqrt(x.float().pow(2).mean(dim=-1, keepdim=True) + self.eps)
+        return normalized.to(source_dtype) * self.weight.to(source_dtype)
+
+
+class RotaryEmbedding(nn.Module):
+    def __init__(self, head_dim: int, theta: float):
+        super().__init__()
+        inverse_frequencies = 1.0 / (theta ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim))
+        self.register_buffer("inverse_frequencies", inverse_frequencies, persistent=False)
+
+    def forward(self, sequence_length: int, device: torch.device, dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor]:
+        positions = torch.arange(sequence_length, device=device, dtype=torch.float32)
+        angles = torch.outer(positions, self.inverse_frequencies.to(device=device))
+        return angles.cos().to(dtype=dtype)[None, None, :, :], angles.sin().to(dtype=dtype)[None, None, :, :]
+
+
+def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
+    even = x[..., 0::2]
+    odd = x[..., 1::2]
+    result = torch.empty_like(x)
+    result[..., 0::2] = even * cos - odd * sin
+    result[..., 1::2] = even * sin + odd * cos
+    return result
+
+
+class SelfAttention(nn.Module):
+    def __init__(self, config: ModelConfig):
+        super().__init__()
+        self.n_head = config.n_head
+        self.head_dim = config.n_embd // config.n_head
+        self.qkv = nn.Linear(config.n_embd, 3 * config.n_embd, bias=False)
+        self.output = nn.Linear(config.n_embd, config.n_embd, bias=False)
+        self.dropout = config.dropout
+
+    def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
+        batch, sequence_length, width = x.shape
+        qkv = self.qkv(x).view(batch, sequence_length, 3, self.n_head, self.head_dim)
+        query, key, value = qkv.unbind(dim=2)
+        query = query.transpose(1, 2)
+        key = key.transpose(1, 2)
+        value = value.transpose(1, 2)
+        query = apply_rope(query, cos, sin)
+        key = apply_rope(key, cos, sin)
+
+        # PyTorch SDPA selects the available fused backend (including Flash when supported).
+        attended = F.scaled_dot_product_attention(
+            query,
+            key,
+            value,
+            attn_mask=None,
+            dropout_p=self.dropout if self.training else 0.0,
+            is_causal=True,
+        )
+        attended = attended.transpose(1, 2).contiguous().view(batch, sequence_length, width)
+        return self.output(attended)
+
+
+class SwiGLU(nn.Module):
+    def __init__(self, width: int, hidden_width: int):
+        super().__init__()
+        self.gate = nn.Linear(width, hidden_width, bias=False)
+        self.up = nn.Linear(width, hidden_width, bias=False)
+        self.down = nn.Linear(hidden_width, width, bias=False)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.down(F.silu(self.gate(x)) * self.up(x))
+
+
+class TransformerBlock(nn.Module):
+    def __init__(self, config: ModelConfig):
+        super().__init__()
+        self.attention_norm = RMSNorm(config.n_embd)
+        self.attention = SelfAttention(config)
+        self.feed_forward_norm = RMSNorm(config.n_embd)
+        self.feed_forward = SwiGLU(config.n_embd, config.ffn_dim)
+        self.residual_dropout = nn.Dropout(config.dropout)
+
+    def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
+        x = x + self.residual_dropout(self.attention(self.attention_norm(x), cos, sin))
+        x = x + self.residual_dropout(self.feed_forward(self.feed_forward_norm(x)))
+        return x
+
+
+class DivCentererLM(nn.Module):
+    def __init__(self, config: ModelConfig):
+        super().__init__()
+        config.validate()
+        self.config = config
+        self.token_embedding = nn.Embedding(config.vocab_size, config.n_embd)
+        self.rotary = RotaryEmbedding(config.n_embd // config.n_head, config.rope_theta)
+        self.layers = nn.ModuleList([TransformerBlock(config) for _ in range(config.n_layer)])
+        self.final_norm = RMSNorm(config.n_embd)
+        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
+        if config.tie_embeddings:
+            self.lm_head.weight = self.token_embedding.weight
+        self.apply(self._initialize_weights)
+
+    @staticmethod
+    def _initialize_weights(module: nn.Module) -> None:
+        if isinstance(module, nn.Linear):
+            nn.init.normal_(module.weight, mean=0.0, std=0.02)
+            if module.bias is not None:
+                nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.Embedding):
+            nn.init.normal_(module.weight, mean=0.0, std=0.02)
+
+    def forward(self, input_ids: torch.Tensor, labels: torch.Tensor | None = None) -> tuple[torch.Tensor, torch.Tensor | None]:
+        _, sequence_length = input_ids.shape
+        if sequence_length > self.config.block_size:
+            raise ValueError(f"Sequence length {sequence_length} exceeds block_size={self.config.block_size}")
+        x = self.token_embedding(input_ids)
+        cos, sin = self.rotary(sequence_length, input_ids.device, x.dtype)
+        for layer in self.layers:
+            x = layer(x, cos, sin)
+        logits = self.lm_head(self.final_norm(x))
+        loss = None
+        if labels is not None:
+            loss = F.cross_entropy(
+                logits.float().reshape(-1, self.config.vocab_size),
+                labels.reshape(-1),
+                ignore_index=-100,
+            )
+        return logits, loss
+
+    def parameter_count(self) -> int:
+        return sum(parameter.numel() for parameter in self.parameters())
+
+    def config_dict(self) -> dict:
+        return asdict(self.config)

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88e2d9f7df927fde0c560d0e2fe52bff47399229f35cac26c8b9b5276bb50370
+size 4874376

tokenizer.json

@@ -0,0 +1,2054 @@
+{
+  "format": "divcenterer-constrained-byte-bpe-v1",
+  "base_vocab_size": 256,
+  "special_tokens": [
+    "<|user|>",
+    "<|assistant|>",
+    "<|end|>"
+  ],
+  "merges": [
+    [
+      32,
+      32
+    ],
+    [
+      10,
+      259
+    ],
+    [
+      110,
+      116
+    ],
+    [
+      58,
+      32
+    ],
+    [
+      105,
+      110
+    ],
+    [
+      100,
+      105
+    ],
+    [
+      98,
+      111
+    ],
+    [
+      101,
+      261
+    ],
+    [
+      104,
+      101
+    ],
+    [
+      115,
+      116
+    ],
+    [
+      100,
+      121
+    ],
+    [
+      103,
+      101
+    ],
+    [
+      60,
+      47
+    ],
+    [
+      265,
+      269
+    ],
+    [
+      101,
+      114
+    ],
+    [
+      264,
+      118
+    ],
+    [
+      105,
+      100
+    ],
+    [
+      108,
+      97
+    ],
+    [
+      97,
+      270
+    ],
+    [
+      266,
+      273
+    ],
+    [
+      268,
+      277
+    ],
+    [
+      108,
+      101
+    ],
+    [
+      97,
+      114
+    ],
+    [
+      105,
+      116
+    ],
+    [
+      99,
+      278
+    ],
+    [
+      10,
+      10
+    ],
+    [
+      34,
+      62
+    ],
+    [
+      61,
+      34
+    ],
+    [
+      272,
+      62
+    ],
+    [
+      116,
+      267
+    ],
+    [
+      111,
+      110
+    ],
+    [
+      105,
+      103
+    ],
+    [
+      112,
+      276
+    ],
+    [
+      262,
+      283
+    ],
+    [
+      292,
+      59
+    ],
+    [
+      116,
+      105
+    ],
+    [
+      67,
+      83
+    ],
+    [
+      295,
+      83
+    ],
+    [
+      60,
+      274
+    ],
+    [
+      271,
+      274
+    ],
+    [
+      297,
+      32
+    ],
+    [
+      298,
+      62
+    ],
+    [
+      48,
+      48
+    ],
+    [
+      282,
+      101
+    ],
+    [
+      72,
+      84
+    ],
+    [
+      77,
+      76
+    ],
+    [
+      303,
+      304
+    ],
+    [
+      104,
+      116
+    ],
+    [
+      290,
+      306
+    ],
+    [
+      263,
+      45
+    ],
+    [
+      97,
+      108
+    ],
+    [
+      267,
+      307
+    ],
+    [
+      109,
+      281
+    ],
+    [
+      286,
+      279
+    ],
+    [
+      108,
+      121
+    ],
+    [
+      114,
+      275
+    ],
+    [
+      103,
+      263
+    ],
+    [
+      49,
+      301
+    ],
+    [
+      45,
+      302
+    ],
+    [
+      48,
+      59
+    ],
+    [
+      60,
+      287
+    ],
+    [
+      99,
+      276
+    ],
+    [
+      104,
+      59
+    ],
+    [
+      109,
+      115
+    ],
+    [
+      109,
+      308
+    ],
+    [
+      115,
+      115
+    ],
+    [
+      115,
+      291
+    ],
+    [
+      118,
+      321
+    ],
+    [
+      121,
+      262
+    ],
+    [
+      260,
+      259
+    ],
+    [
+      262,
+      316
+    ],
+    [
+      262,
+      318
+    ],
+    [
+      264,
+      325
+    ],
+    [
+      271,
+      287
+    ],
+    [
+      280,
+      120
+    ],
+    [
+      299,
+      275
+    ],
+    [
+      310,
+      329
+    ],
+    [
+      311,
+      315
+    ],
+    [
+      312,
+      285
+    ],
+    [
+      317,
+      322
+    ],
+    [
+      320,
+      324
+    ],
+    [
+      323,
+      335
+    ],
+    [
+      331,
+      327
+    ],
+    [
+      336,
+      330
+    ],
+    [
+      338,
+      293
+    ],
+    [
+      339,
+      337
+    ],
+    [
+      340,
+      326
+    ],
+    [
+      111,
+      114
+    ],
+    [
+      97,
+      110
+    ],
+    [
+      115,
+      101
+    ],
+    [
+      111,
+      261
+    ],
+    [
+      334,
+      286
+    ],
+    [
+      45,
+      49
+    ],
+    [
+      45,
+      48
+    ],
+    [
+      32,
+      344
+    ],
+    [
+      99,
+      349
+    ],
+    [
+      105,
+      101
+    ],
+    [
+      294,
+      99
+    ],
+    [
+      109,
+      97
+    ],
+    [
+      294,
+      289
+    ],
+    [
+      289,
+      313
+    ],
+    [
+      265,
+      120
+    ],
+    [
+      354,
+      266
+    ],
+    [
+      263,
+      103
+    ],
+    [
+      99,
+      358
+    ],
+    [
+      101,
+      116
+    ],
+    [
+      119,
+      282
+    ],
+    [
+      365,
+      104
+    ],
+    [
+      117,
+      115
+    ],
+    [
+      117,
+      268
+    ],
+    [
+      347,
+      100
+    ],
+    [
+      116,
+      104
+    ],
+    [
+      290,
+      110
+    ],
+    [
+      309,
+      371
+    ],
+    [
+      112,
+      346
+    ],
+    [
+      118,
+      355
+    ],
+    [
+      119,
+      373
+    ],
+    [
+      374,
+      375
+    ],
+    [
+      376,
+      116
+    ],
+    [
+      106,
+      368
+    ],
+    [
+      117,
+      116
+    ],
+    [
+      281,
+      356
+    ],
+    [
+      380,
+      280
+    ],
+    [
+      348,
+      363
+    ],
+    [
+      45,
+      361
+    ],
+    [
+      70,
+      333
+    ],
+    [
+      71,
+      314
+    ],
+    [
+      99,
+      101
+    ],
+    [
+      102,
+      121
+    ],
+    [
+      102,
+      333
+    ],
+    [
+      103,
+      314
+    ],
+    [
+      105,
+      387
+    ],
+    [
+      291,
+      386
+    ],
+    [
+      341,
+      388
+    ],
+    [
+      341,
+      389
+    ],
+    [
+      372,
+      343
+    ],
+    [
+      378,
+      390
+    ],
+    [
+      383,
+      293
+    ],
+    [
+      384,
+      360
+    ],
+    [
+      391,
+      343
+    ],
+    [
+      392,
+      59
+    ],
+    [
+      393,
+      59
+    ],
+    [
+      395,
+      396
+    ],
+    [
+      357,
+      263
+    ],
+    [
+      270,
+      116
+    ],
+    [
+      114,
+      111
+    ],
+    [
+      105,
+      115
+    ],
+    [
+      367,
+      362
+    ],
+    [
+      309,
+      313
+    ],
+    [
+      117,
+      112
+    ],
+    [
+      117,
+      114
+    ],
+    [
+      114,
+      101
+    ],
+    [
+      97,
+      116
+    ],
+    [
+      364,
+      409
+    ],
+    [
+      412,
+      110
+    ],
+    [
+      104,
+      346
+    ],
+    [
+      116,
+      281
+    ],
+    [
+      415,
+      403
+    ],
+    [
+      101,
+      108
+    ],
+    [
+      118,
+      273
+    ],
+    [
+      60,
+      381
+    ],
+    [
+      271,
+      381
+    ],
+    [
+      419,
+      353
+    ],
+    [
+      420,
+      62
+    ],
+    [
+      60,
+      382
+    ],
+    [
+      271,
+      382
+    ],
+    [
+      423,
+      353
+    ],
+    [
+      424,
+      62
+    ],
+    [
+      85,
+      348
+    ],
+    [
+      105,
+      122
+    ],
+    [
+      299,
+      344
+    ],
+    [
+      60,
+      402
+    ],
+    [
+      271,
+      402
+    ],
+    [
+      430,
+      353
+    ],
+    [
+      431,
+      62
+    ],
+    [
+      107,
+      101
+    ],
+    [
+      282,
+      115
+    ],
+    [
+      101,
+      100
+    ],
+    [
+      67,
+      278
+    ],
+    [
+      111,
+      102
+    ],
+    [
+      99,
+      116
+    ],
+    [
+      101,
+      280
+    ],
+    [
+      101,
+      110
+    ],
+    [
+      114,
+      413
+    ],
+    [
+      108,
+      108
+    ],
+    [
+      265,
+      370
+    ],
+    [
+      121,
+      280
+    ],
+    [
+      112,
+      347
+    ],
+    [
+      349,
+      407
+    ],
+    [
+      356,
+      407
+    ],
+    [
+      414,
+      428
+    ],
+    [
+      418,
+      448
+    ],
+    [
+      449,
+      447
+    ],
+    [
+      112,
+      379
+    ],
+    [
+      84,
+      267
+    ],
+    [
+      112,
+      281
+    ],
+    [
+      454,
+      266
+    ],
+    [
+      105,
+      118
+    ],
+    [
+      109,
+      266
+    ],
+    [
+      440,
+      457
+    ],
+    [
+      446,
+      417
+    ],
+    [
+      111,
+      99
+    ],
+    [
+      97,
+      434
+    ],
+    [
+      116,
+      111
+    ],
+    [
+      119,
+      275
+    ],
+    [
+      463,
+      403
+    ],
+    [
+      97,
+      348
+    ],
+    [
+      280,
+      465
+    ],
+    [
+      99,
+      281
+    ],
+    [
+      467,
+      100
+    ],
+    [
+      268,
+      445
+    ],
+    [
+      108,
+      355
+    ],
+    [
+      112,
+      470
+    ],
+    [
+      115,
+      408
+    ],
+    [
+      471,
+      100
+    ],
+    [
+      472,
+      473
+    ],
+    [
+      114,
+      302
+    ],
+    [
+      110,
+      111
+    ],
+    [
+      370,
+      411
+    ],
+    [
+      97,
+      112
+    ],
+    [
+      82,
+      413
+    ],
+    [
+      109,
+      275
+    ],
+    [
+      379,
+      452
+    ],
+    [
+      115,
+      111
+    ],
+    [
+      111,
+      119
+    ],
+    [
+      115,
+      99
+    ],
+    [
+      354,
+      404
+    ],
+    [
+      485,
+      108
+    ],
+    [
+      99,
+      414
+    ],
+    [
+      347,
+      487
+    ],
+    [
+      410,
+      441
+    ],
+    [
+      49,
+      55
+    ],
+    [
+      98,
+      106
+    ],
+    [
+      101,
+      439
+    ],
+    [
+      111,
+      491
+    ],
+    [
+      493,
+      492
+    ],
+    [
+      357,
+      308
+    ],
+    [
+      102,
+      460
+    ],
+    [
+      370,
+      362
+    ],
+    [
+      496,
+      367
+    ],
+    [
+      49,
+      56
+    ],
+    [
+      49,
+      54
+    ],
+    [
+      49,
+      50
+    ],
+    [
+      49,
+      57
+    ],
+    [
+      267,
+      404
+    ],
+    [
+      49,
+      51
+    ],
+    [
+      49,
+      52
+    ],
+    [
+      48,
+      51
+    ],
+    [
+      48,
+      52
+    ],
+    [
+      49,
+      53
+    ],
+    [
+      377,
+      45
+    ],
+    [
+      110,
+      101
+    ],
+    [
+      111,
+      117
+    ],
+    [
+      48,
+      50
+    ],
+    [
+      48,
+      54
+    ],
+    [
+      77,
+      461
+    ],
+    [
+      469,
+      115
+    ],
+    [
+      366,
+      263
+    ],
+    [
+      264,
+      410
+    ],
+    [
+      351,
+      50
+    ],
+    [
+      363,
+      115
+    ],
+    [
+      517,
+      519
+    ],
+    [
+      80,
+      379
+    ],
+    [
+      275,
+      101
+    ],
+    [
+      476,
+      116
+    ],
+    [
+      484,
+      489
+    ],
+    [
+      351,
+      55
+    ],
+    [
+      80,
+      466
+    ],
+    [
+      87,
+      475
+    ],
+    [
+      283,
+      115
+    ],
+    [
+      107,
+      408
+    ],
+    [
+      311,
+      529
+    ],
+    [
+      48,
+      56
+    ],
+    [
+      48,
+      55
+    ],
+    [
+      71,
+      456
+    ],
+    [
+      100,
+      280
+    ],
+    [
+      480,
+      534
+    ],
+    [
+      533,
+      101
+    ],
+    [
+      45,
+      301
+    ],
+    [
+      283,
+      436
+    ],
+    [
+      79,
+      481
+    ],
+    [
+      102,
+      105
+    ],
+    [
+      112,
+      404
+    ],
+    [
+      540,
+      443
+    ],
+    [
+      541,
+      118
+    ],
+    [
+      351,
+      51
+    ],
+    [
+      351,
+      57
+    ],
+    [
+      49,
+      285
+    ],
+    [
+      352,
+      55
+    ],
+    [
+      48,
+      285
+    ],
+    [
+      53,
+      285
+    ],
+    [
+      351,
+      56
+    ],
+    [
+      352,
+      53
+    ],
+    [
+      48,
+      57
+    ],
+    [
+      351,
+      52
+    ],
+    [
+      352,
+      52
+    ],
+    [
+      352,
+      54
+    ],
+    [
+      352,
+      56
+    ],
+    [
+      352,
+      50
+    ],
+    [
+      351,
+      54
+    ],
+    [
+      352,
+      51
+    ],
+    [
+      101,
+      112
+    ],
+    [
+      351,
+      53
+    ],
+    [
+      52,
+      285
+    ],
+    [
+      352,
+      57
+    ],
+    [
+      478,
+      112
+    ],
+    [
+      54,
+      285
+    ],
+    [
+      48,
+      53
+    ],
+    [
+      50,
+      285
+    ],
+    [
+      55,
+      285
+    ],
+    [
+      108,
+      117
+    ],
+    [
+      99,
+      408
+    ],
+    [
+      100,
+      100
+    ],
+    [
+      102,
+      117
+    ],
+    [
+      51,
+      285
+    ],
+    [
+      56,
+      285
+    ],
+    [
+      97,
+      439
+    ],
+    [
+      57,
+      285
+    ],
+    [
+      108,
+      100
+    ],
+    [
+      350,
+      459
+    ],
+    [
+      113,
+      117
+    ],
+    [
+      114,
+      405
+    ],
+    [
+      334,
+      312
+    ],
+    [
+      350,
+      464
+    ],
+    [
+      350,
+      486
+    ],
+    [
+      350,
+      488
+    ],
+    [
+      109,
+      111
+    ],
+    [
+      350,
+      494
+    ],
+    [
+      99,
+      111
+    ],
+    [
+      350,
+      416
+    ],
+    [
+      350,
+      458
+    ],
+    [
+      350,
+      495
+    ],
+    [
+      350,
+      361
+    ],
+    [
+      350,
+      497
+    ],
+    [
+      350,
+      498
+    ],
+    [
+      99,
+      461
+    ],
+    [
+      97,
+      268
+    ],
+    [
+      350,
+      283
+    ],
+    [
+      350,
+      360
+    ],
+    [
+      350,
+      468
+    ],
+    [
+      105,
+      99
+    ],
+    [
+      350,
+      503
+    ],
+    [
+      99,
+      108
+    ],
+    [
+      601,
+      511
+    ],
+    [
+      112,
+      466
+    ],
+    [
+      509,
+      310
+    ],
+    [
+      115,
+      522
+    ],
+    [
+      263,
+      605
+    ],
+    [
+      75,
+      101
+    ],
+    [
+      103,
+      456
+    ],
+    [
+      117,
+      309
+    ],
+    [
+      118,
+      405
+    ],
+    [
+      607,
+      560
+    ],
+    [
+      608,
+      441
+    ],
+    [
+      610,
+      609
+    ],
+    [
+      83,
+      364
+    ],
+    [
+      97,
+      115
+    ],
+    [
+      102,
+      346
+    ],
+    [
+      115,
+      428
+    ],
+    [
+      283,
+      362
+    ],
+    [
+      509,
+      617
+    ],
+    [
+      619,
+      436
+    ],
+    [
+      68,
+      111
+    ],
+    [
+      98,
+      404
+    ],
+    [
+      100,
+      483
+    ],
+    [
+      111,
+      481
+    ],
+    [
+      115,
+      273
+    ],
+    [
+      119,
+      263
+    ],
+    [
+      119,
+      475
+    ],
+    [
+      119,
+      625
+    ],
+    [
+      410,
+      627
+    ],
+    [
+      622,
+      628
+    ],
+    [
+      626,
+      623
+    ],
+    [
+      267,
+      364
+    ],
+    [
+      515,
+      632
+    ],
+    [
+      105,
+      261
+    ],
+    [
+      111,
+      634
+    ],
+    [
+      112,
+      289
+    ],
+    [
+      112,
+      635
+    ],
+    [
+      115,
+      636
+    ],
+    [
+      410,
+      638
+    ],
+    [
+      480,
+      637
+    ],
+    [
+      639,
+      100
+    ],
+    [
+      101,
+      281
+    ],
+    [
+      357,
+      434
+    ],
+    [
+      564,
+      642
+    ],
+    [
+      73,
+      116
+    ],
+    [
+      97,
+      120
+    ],
+    [
+      97,
+      121
+    ],
+    [
+      101,
+      115
+    ],
+    [
+      109,
+      101
+    ],
+    [
+      117,
+      348
+    ],
+    [
+      119,
+      647
+    ],
+    [
+      543,
+      522
+    ],
+    [
+      645,
+      115
+    ],
+    [
+      646,
+      648
+    ],
+    [
+      651,
+      115
+    ],
+    [
+      99,
+      104
+    ],
+    [
+      105,
+      112
+    ],
+    [
+      110,
+      657
+    ],
+    [
+      112,
+      364
+    ],
+    [
+      112,
+      411
+    ],
+    [
+      115,
+      658
+    ],
+    [
+      289,
+      101
+    ],
+    [
+      370,
+      405
+    ],
+    [
+      660,
+      656
+    ],
+    [
+      661,
+      659
+    ],
+    [
+      65,
+      571
+    ],
+    [
+      97,
+      100
+    ],
+    [
+      97,
+      105
+    ],
+    [
+      97,
+      118
+    ],
+    [
+      98,
+      280
+    ],
+    [
+      98,
+      417
+    ],
+    [
+      100,
+      101
+    ],
+    [
+      112,
+      277
+    ],
+    [
+      114,
+      117
+    ],
+    [
+      115,
+      119
+    ],
+    [
+      276,
+      670
+    ],
+    [
+      280,
+      115
+    ],
+    [
+      347,
+      675
+    ],
+    [
+      668,
+      676
+    ],
+    [
+      669,
+      679
+    ],
+    [
+      671,
+      483
+    ],
+    [
+      672,
+      667
+    ],
+    [
+      674,
+      677
+    ],
+    [
+      678,
+      273
+    ],
+    [
+      83,
+      116
+    ],
+    [
+      98,
+      121
+    ],
+    [
+      101,
+      120
+    ],
+    [
+      104,
+      511
+    ],
+    [
+      115,
+      435
+    ],
+    [
+      115,
+      688
+    ],
+    [
+      460,
+      570
+    ],
+    [
+      685,
+      445
+    ],
+    [
+      687,
+      575
+    ],
+    [
+      690,
+      577
+    ],
+    [
+      691,
+      121
+    ],
+    [
+      510,
+      436
+    ],
+    [
+      45,
+      310
+    ],
+    [
+      105,
+      109
+    ],
+    [
+      109,
+      263
+    ],
+    [
+      275,
+      436
+    ],
+    [
+      443,
+      697
+    ],
+    [
+      482,
+      569
+    ],
+    [
+      542,
+      362
+    ],
+    [
+      543,
+      700
+    ],
+    [
+      572,
+      701
+    ],
+    [
+      698,
+      309
+    ],
+    [
+      699,
+      706
+    ],
+    [
+      702,
+      358
+    ],
+    [
+      45,
+      50
+    ],
+    [
+      49,
+      49
+    ],
+    [
+      265,
+      116
+    ],
+    [
+      404,
+      711
+    ],
+    [
+      101,
+      478
+    ],
+    [
+      111,
+      116
+    ],
+    [
+      115,
+      438
+    ],
+    [
+      116,
+      713
+    ],
+    [
+      355,
+      116
+    ],
+    [
+      579,
+      717
+    ],
+    [
+      715,
+      116
+    ],
+    [
+      716,
+      714
+    ],
+    [
+      98,
+      282
+    ],
+    [
+      101,
+      277
+    ],
+    [
+      110,
+      281
+    ],
+    [
+      346,
+      721
+    ],
+    [
+      569,
+      723
+    ],
+    [
+      722,
+      114
+    ],
+    [
+      97,
+      109
+    ],
+    [
+      98,
+      273
+    ],
+    [
+      104,
+      111
+    ],
+    [
+      112,
+      580
+    ],
+    [
+      114,
+      478
+    ],
+    [
+      346,
+      347
+    ],
+    [
+      443,
+      483
+    ],
+    [
+      727,
+      728
+    ],
+    [
+      729,
+      733
+    ],
+    [
+      730,
+      109
+    ],
+    [
+      731,
+      275
+    ],
+    [
+      732,
+      270
+    ],
+    [
+      97,
+      103
+    ],
+    [
+      98,
+      314
+    ],
+    [
+      98,
+      368
+    ],
+    [
+      100,
+      114
+    ],
+    [
+      105,
+      114
+    ],
+    [
+      108,
+      302
+    ],
+    [
+      109,
+      364
+    ],
+    [
+      111,
+      370
+    ],
+    [
+      115,
+      309
+    ],
+    [
+      115,
+      411
+    ],
+    [
+      115,
+      579
+    ],
+    [
+      115,
+      585
+    ],
+    [
+      116,
+      121
+    ],
+    [
+      404,
+      741
+    ],
+    [
+      410,
+      108
+    ],
+    [
+      417,
+      744
+    ],
+    [
+      587,
+      745
+    ],
+    [
+      739,
+      289
+    ],
+    [
+      740,
+      270
+    ],
+    [
+      742,
+      756
+    ],
+    [
+      743,
+      753
+    ],
+    [
+      747,
+      751
+    ],
+    [
+      748,
+      754
+    ],
+    [
+      749,
+      759
+    ],
+    [
+      750,
+      746
+    ],
+    [
+      98,
+      379
+    ],
+    [
+      98,
+      580
+    ],
+    [
+      99,
+      280
+    ],
+    [
+      103,
+      266
+    ]
+  ],
+  "vocab_size": 768,
+  "requested_vocab_size": 768,
+  "min_pair_frequency": 2,
+  "atomizer": "syntax-aware-html-css-segments-v1",
+  "merge_policy": "BPE merges stay within prose lexemes, whitespace runs, complete HTML tags, or recognized CSS declarations.",
+  "training_rows": 3276,
+  "training_note": "Trained on the train split only; merges cannot cross pretokenized HTML/CSS atoms."
+}

tokenizer.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Iterable
+
+BASE_VOCAB_SIZE = 256
+ATOM_RE = re.compile(
+    r"<!--[\s\S]*?-->|<!doctype\s+html>|</?[A-Za-z][^<>]*>|"
+    r"\b(?:display|place-items|align-items|justify-content|min-height|margin|font|padding|border|border-radius|background)\s*:\s*[^;{}]+;?|"
+    r"\s+|[A-Za-z_$][A-Za-z0-9_$-]*|[0-9]+(?:\.[0-9]+)?(?:%|px|rem|em|vh|vw)?|[^\x00-\x7F]|[^\w\s]",
+    re.IGNORECASE | re.ASCII,
+)
+
+
+class ConstrainedBPE:
+    """Byte-fallback BPE matching DivCenterer's syntax-aware Node tokenizer."""
+
+    def __init__(self, model: dict):
+        if model.get("format") != "divcenterer-constrained-byte-bpe-v1":
+            raise ValueError("Unrecognized tokenizer model format")
+        self.special_tokens: list[str] = model["special_tokens"]
+        self.special_to_id = {
+            token: BASE_VOCAB_SIZE + index
+            for index, token in enumerate(self.special_tokens)
+        }
+        self.special_by_id = {value: key for key, value in self.special_to_id.items()}
+        self.merge_ranks: dict[tuple[int, int], int] = {}
+        self.token_bytes: list[bytes | None] = [bytes([index]) for index in range(BASE_VOCAB_SIZE)]
+        self.token_bytes.extend([None] * len(self.special_tokens))
+        for rank, (left, right) in enumerate(model["merges"]):
+            if self.token_bytes[left] is None or self.token_bytes[right] is None:
+                raise ValueError(f"Merge {rank} refers to a special or missing token")
+            token_id = BASE_VOCAB_SIZE + len(self.special_tokens) + rank
+            self.merge_ranks[(left, right)] = rank
+            self.token_bytes.append(self.token_bytes[left] + self.token_bytes[right])
+        self._special_re = re.compile("(" + "|".join(re.escape(t) for t in self.special_tokens) + ")")
+        self._atom_cache: dict[str, list[int]] = {}
+
+    @classmethod
+    def from_file(cls, filename: str | Path) -> "ConstrainedBPE":
+        with Path(filename).open("r", encoding="utf-8") as stream:
+            return cls(json.load(stream))
+
+    def _atoms(self, text: str) -> list[str]:
+        atoms: list[str] = []
+        cursor = 0
+        for match in ATOM_RE.finditer(text):
+            if match.start() != cursor:
+                raise ValueError(f"Pretokenizer left a gap at character {cursor}")
+            atoms.append(match.group(0))
+            cursor = match.end()
+        if cursor != len(text):
+            raise ValueError(f"Pretokenizer left a gap at character {cursor}")
+        return atoms
+
+    @staticmethod
+    def _apply_merge(ids: list[int], left: int, right: int, merged: int) -> list[int]:
+        result: list[int] = []
+        index = 0
+        while index < len(ids):
+            if index + 1 < len(ids) and ids[index] == left and ids[index + 1] == right:
+                result.append(merged)
+                index += 2
+            else:
+                result.append(ids[index])
+                index += 1
+        return result
+
+    def _encode_atom(self, atom: str) -> list[int]:
+        cached = self._atom_cache.get(atom)
+        if cached is not None:
+            return cached.copy()
+        ids = list(atom.encode("utf-8"))
+        while len(ids) > 1:
+            best_rank: int | None = None
+            best_pair: tuple[int, int] | None = None
+            for left, right in zip(ids, ids[1:]):
+                rank = self.merge_ranks.get((left, right))
+                if rank is not None and (best_rank is None or rank < best_rank):
+                    best_rank = rank
+                    best_pair = (left, right)
+            if best_pair is None or best_rank is None:
+                break
+            merged_id = BASE_VOCAB_SIZE + len(self.special_tokens) + best_rank
+            ids = self._apply_merge(ids, best_pair[0], best_pair[1], merged_id)
+        self._atom_cache[atom] = ids.copy()
+        return ids
+
+    def encode(self, text: str) -> list[int]:
+        result: list[int] = []
+        for part in self._special_re.split(text):
+            if not part:
+                continue
+            special_id = self.special_to_id.get(part)
+            if special_id is not None:
+                result.append(special_id)
+            else:
+                for atom in self._atoms(part):
+                    result.extend(self._encode_atom(atom))
+        return result
+
+    def decode(self, ids: Iterable[int]) -> str:
+        output: list[str] = []
+        byte_buffer = bytearray()
+
+        def flush_bytes() -> None:
+            if byte_buffer:
+                output.append(bytes(byte_buffer).decode("utf-8", errors="replace"))
+                byte_buffer.clear()
+
+        for token_id in ids:
+            special = self.special_by_id.get(int(token_id))
+            if special is not None:
+                flush_bytes()
+                output.append(special)
+            elif 0 <= int(token_id) < len(self.token_bytes) and self.token_bytes[int(token_id)] is not None:
+                byte_buffer.extend(self.token_bytes[int(token_id)])
+            else:
+                raise ValueError(f"Unknown token id: {token_id}")
+        flush_bytes()
+        return "".join(output)
+
+    def special_id(self, token: str) -> int:
+        return self.special_to_id[token]
+
+    @property
+    def vocab_size(self) -> int:
+        return len(self.token_bytes)
+
+
+def conversation_prefix(prompt: str) -> str:
+    return f"<|user|>\n{prompt}\n<|assistant|>\n"
+
+
+def conversation_text(prompt: str, answer: str) -> str:
+    return f"{conversation_prefix(prompt)}{answer}\n<|end|>"