Flint-S / flint_s_decode.py
rgup's picture
Publish the stronger-agent Flint S 13.3 candidate and quantization comparisons
2d93c43 verified
Raw History Blame Contribute Delete
5.23 kB
"""Reference decoder of a Flint S package: every quantized weight back to the dense bf16 tensor. Needs torch + safetensors.
Package (model-LL.safetensors per layer LL, flint_s.json):
X.codes uint8 [rows, bytes] per-row bitstream of fields, each MSB first, the row zero padded to a byte
X.scales f32 [rows] (trellis) or [rows, columns / 128] (int8)
every other tensor: bf16, used as is.
shared.safetensors: codebook.v4 f32 [65536, 4], codebook.v2 f32 [65536, 2], rotation.<columns>.signs f32 [columns],
rotation.<columns>.q f32 [m, m].
flint_s.json "quantized": {X: [format, rows, columns]}, format one of
T6/T7/T8 trellis, 4 weights per step, restart every 64 columns: each block is a 16-bit start state then 15 T-bit symbols,
state_t = ((state_{t-1} << T) | symbol_t) & 0xFFFF, weights = codebook.v4[state]
k3/k4 trellis, 2 weights per step, one tape per row: a 16-bit start state then (columns / 2 - 1) symbols of 2k bits,
the same state update, weights = codebook.v2[state]
int8 one byte b per weight, weight = 2b - 255, one scale per (row, 128 columns)
Dense: W = ((weights * scales) @ R) * signs, R = kron(H, q): H the orthonormal Sylvester Hadamard of size columns / m, q the
stored m x m orthogonal (m = the odd part of columns; R = H when m = 1).
usage: python flint_s_decode.py PACKAGE_DIR OUT_DIR (one dense safetensors per package shard)"""
import json
import sys
from functools import cache
from pathlib import Path
import torch
from safetensors import safe_open
from safetensors.torch import load_file, save_file
def layout(fmt: str, columns: int) -> tuple[list[int], list[bool]]:
"""Field widths of one row's bitstream and which fields are start states."""
if fmt[0] == "T":
widths = ([16] + [int(fmt[1:])] * 15) * (columns // 64)
else:
widths = [16] + [2 * int(fmt[1:])] * (columns // 2 - 1)
return widths, [w == 16 for w in widths]
@cache
def bit_map(widths: tuple[int, ...]) -> tuple[torch.Tensor, torch.Tensor]:
"""For every bit of a row: the field it belongs to and its place value in that field (MSB first)."""
field = torch.repeat_interleave(torch.arange(len(widths)), torch.tensor(widths))
place = torch.cat([torch.arange(w - 1, -1, -1) for w in widths])
return field, place
def unpack(codes: torch.Tensor, widths: list[int]) -> torch.Tensor:
"""uint8 [rows, bytes] -> int64 [rows, fields]."""
field, place = bit_map(tuple(widths))
bits = ((codes[:, :, None] >> torch.arange(7, -1, -1, dtype=torch.uint8)) & 1).reshape(codes.shape[0], -1)
assert bits.shape[1] - len(field) in range(8), (bits.shape, len(field))
return torch.zeros(codes.shape[0], len(widths), dtype=torch.int64).index_add_(1, field, bits[:, : len(field)].long() << place)
def units(fmt: str, codes: torch.Tensor, rows: int, columns: int, tables: dict[str, torch.Tensor]) -> torch.Tensor:
"""The rotated-basis weights in scale units, fp32 [rows, columns]."""
if fmt == "int8":
return 2 * codes.float() - 255
widths, starts = layout(fmt, columns)
fields = unpack(codes, widths)
states = fields.clone()
for t in range(1, len(widths)):
if not starts[t]:
states[:, t] = ((states[:, t - 1] << widths[t]) | fields[:, t]) & 0xFFFF
return tables["v4" if fmt[0] == "T" else "v2"][states].reshape(rows, columns)
@cache
def hadamard(n: int) -> torch.Tensor:
h = torch.ones(1, 1)
while h.shape[0] < n:
h = torch.cat([torch.cat([h, h], 1), torch.cat([h, -h], 1)])
return h / n**0.5
def dense(fmt: str, codes: torch.Tensor, scales: torch.Tensor, rows: int, columns: int, shared: dict[str, torch.Tensor]) -> torch.Tensor:
u = units(fmt, codes, rows, columns, {"v4": shared["codebook.v4"], "v2": shared["codebook.v2"]})
if fmt == "int8":
w_rot = (u.reshape(rows, -1, 128) * scales[..., None]).reshape(rows, columns)
else:
w_rot = u * scales[:, None]
m = columns
while m % 2 == 0:
m //= 2
r = hadamard(columns) if m == 1 else torch.kron(hadamard(columns // m), shared[f"rotation.{columns}.q"])
return ((w_rot @ r) * shared[f"rotation.{columns}.signs"]).bfloat16()
def decode_shard(path: Path, quantized: dict[str, list], shared: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
out = {}
with safe_open(path, "pt") as f:
for k in f.keys():
if k.endswith(".scales"):
continue
if k.endswith(".codes"):
name = k.removesuffix(".codes")
fmt, rows, columns = quantized[name]
out[name] = dense(fmt, f.get_tensor(k), f.get_tensor(name + ".scales"), rows, columns, shared)
else:
out[k] = f.get_tensor(k)
return out
if __name__ == "__main__":
package, out_dir = Path(sys.argv[1]), Path(sys.argv[2])
out_dir.mkdir(parents=True, exist_ok=True)
quantized = json.loads((package / "flint_s.json").read_text())["quantized"]
shared = load_file(package / "shared.safetensors")
for shard in sorted(package.glob("model-*.safetensors")):
save_file(decode_shard(shard, quantized, shared), out_dir / shard.name)
print(shard.name, flush=True)