Instructions to use trymirai/Flint-S with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Trellis
How to use trymirai/Flint-S with Trellis:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
Download flint_s_decode.py from trymirai/Flint-S: direct link, hf CLI and curl.
- Browser
- Download file 5.23 kB
-
https://huggingface.co/trymirai/Flint-S/resolve/main/flint_s_decode.py
- Command line
-
hf download hf://trymirai/Flint-S/flint_s_decode.py
-
curl -L -o flint_s_decode.py https://huggingface.co/trymirai/Flint-S/resolve/main/flint_s_decode.py
5.23 kB
| """Reference decoder of a Flint S package: every quantized weight back to the dense bf16 tensor. Needs torch + safetensors. | |
| Package (model-LL.safetensors per layer LL, flint_s.json): | |
| X.codes uint8 [rows, bytes] per-row bitstream of fields, each MSB first, the row zero padded to a byte | |
| X.scales f32 [rows] (trellis) or [rows, columns / 128] (int8) | |
| every other tensor: bf16, used as is. | |
| shared.safetensors: codebook.v4 f32 [65536, 4], codebook.v2 f32 [65536, 2], rotation.<columns>.signs f32 [columns], | |
| rotation.<columns>.q f32 [m, m]. | |
| flint_s.json "quantized": {X: [format, rows, columns]}, format one of | |
| T6/T7/T8 trellis, 4 weights per step, restart every 64 columns: each block is a 16-bit start state then 15 T-bit symbols, | |
| state_t = ((state_{t-1} << T) | symbol_t) & 0xFFFF, weights = codebook.v4[state] | |
| k3/k4 trellis, 2 weights per step, one tape per row: a 16-bit start state then (columns / 2 - 1) symbols of 2k bits, | |
| the same state update, weights = codebook.v2[state] | |
| int8 one byte b per weight, weight = 2b - 255, one scale per (row, 128 columns) | |
| Dense: W = ((weights * scales) @ R) * signs, R = kron(H, q): H the orthonormal Sylvester Hadamard of size columns / m, q the | |
| stored m x m orthogonal (m = the odd part of columns; R = H when m = 1). | |
| usage: python flint_s_decode.py PACKAGE_DIR OUT_DIR (one dense safetensors per package shard)""" | |
| import json | |
| import sys | |
| from functools import cache | |
| from pathlib import Path | |
| import torch | |
| from safetensors import safe_open | |
| from safetensors.torch import load_file, save_file | |
| def layout(fmt: str, columns: int) -> tuple[list[int], list[bool]]: | |
| """Field widths of one row's bitstream and which fields are start states.""" | |
| if fmt[0] == "T": | |
| widths = ([16] + [int(fmt[1:])] * 15) * (columns // 64) | |
| else: | |
| widths = [16] + [2 * int(fmt[1:])] * (columns // 2 - 1) | |
| return widths, [w == 16 for w in widths] | |
| def bit_map(widths: tuple[int, ...]) -> tuple[torch.Tensor, torch.Tensor]: | |
| """For every bit of a row: the field it belongs to and its place value in that field (MSB first).""" | |
| field = torch.repeat_interleave(torch.arange(len(widths)), torch.tensor(widths)) | |
| place = torch.cat([torch.arange(w - 1, -1, -1) for w in widths]) | |
| return field, place | |
| def unpack(codes: torch.Tensor, widths: list[int]) -> torch.Tensor: | |
| """uint8 [rows, bytes] -> int64 [rows, fields].""" | |
| field, place = bit_map(tuple(widths)) | |
| bits = ((codes[:, :, None] >> torch.arange(7, -1, -1, dtype=torch.uint8)) & 1).reshape(codes.shape[0], -1) | |
| assert bits.shape[1] - len(field) in range(8), (bits.shape, len(field)) | |
| return torch.zeros(codes.shape[0], len(widths), dtype=torch.int64).index_add_(1, field, bits[:, : len(field)].long() << place) | |
| def units(fmt: str, codes: torch.Tensor, rows: int, columns: int, tables: dict[str, torch.Tensor]) -> torch.Tensor: | |
| """The rotated-basis weights in scale units, fp32 [rows, columns].""" | |
| if fmt == "int8": | |
| return 2 * codes.float() - 255 | |
| widths, starts = layout(fmt, columns) | |
| fields = unpack(codes, widths) | |
| states = fields.clone() | |
| for t in range(1, len(widths)): | |
| if not starts[t]: | |
| states[:, t] = ((states[:, t - 1] << widths[t]) | fields[:, t]) & 0xFFFF | |
| return tables["v4" if fmt[0] == "T" else "v2"][states].reshape(rows, columns) | |
| def hadamard(n: int) -> torch.Tensor: | |
| h = torch.ones(1, 1) | |
| while h.shape[0] < n: | |
| h = torch.cat([torch.cat([h, h], 1), torch.cat([h, -h], 1)]) | |
| return h / n**0.5 | |
| def dense(fmt: str, codes: torch.Tensor, scales: torch.Tensor, rows: int, columns: int, shared: dict[str, torch.Tensor]) -> torch.Tensor: | |
| u = units(fmt, codes, rows, columns, {"v4": shared["codebook.v4"], "v2": shared["codebook.v2"]}) | |
| if fmt == "int8": | |
| w_rot = (u.reshape(rows, -1, 128) * scales[..., None]).reshape(rows, columns) | |
| else: | |
| w_rot = u * scales[:, None] | |
| m = columns | |
| while m % 2 == 0: | |
| m //= 2 | |
| r = hadamard(columns) if m == 1 else torch.kron(hadamard(columns // m), shared[f"rotation.{columns}.q"]) | |
| return ((w_rot @ r) * shared[f"rotation.{columns}.signs"]).bfloat16() | |
| def decode_shard(path: Path, quantized: dict[str, list], shared: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: | |
| out = {} | |
| with safe_open(path, "pt") as f: | |
| for k in f.keys(): | |
| if k.endswith(".scales"): | |
| continue | |
| if k.endswith(".codes"): | |
| name = k.removesuffix(".codes") | |
| fmt, rows, columns = quantized[name] | |
| out[name] = dense(fmt, f.get_tensor(k), f.get_tensor(name + ".scales"), rows, columns, shared) | |
| else: | |
| out[k] = f.get_tensor(k) | |
| return out | |
| if __name__ == "__main__": | |
| package, out_dir = Path(sys.argv[1]), Path(sys.argv[2]) | |
| out_dir.mkdir(parents=True, exist_ok=True) | |
| quantized = json.loads((package / "flint_s.json").read_text())["quantized"] | |
| shared = load_file(package / "shared.safetensors") | |
| for shard in sorted(package.glob("model-*.safetensors")): | |
| save_file(decode_shard(shard, quantized, shared), out_dir / shard.name) | |
| print(shard.name, flush=True) | |