Samonte's picture
slm A-s00080
698535a verified
Raw History Blame Contribute Delete
10.3 kB
"""What a smaller model buys, by the arena's own formulas, before any GPU time is spent.
Three things move when the model shrinks:
SLM pool E = 0.4 e_TTFT + 0.3 e_mem + 0.2 e_N + 0.1 e_TPOT against the base, score x (1 + 0.25 E)
cost the system and escalation pools price every SLM token by parameter class
(definition r_class: <= 1B 0.02, <= 3B 0.05, else 0.1 micro-dollars per token),
against frontier answers at their list prices (Claude Haiku 5.5 0.1/0.5, GLM 5.3 Flash
0.048/0.2 dollars per million tokens in/out)
speed prefill FLOPs (first token time) and the bytes read per decoded token (time per token)
Parameters and computed memory are exact (safetensors headers, the validators' formula). The speed
ratios are first order estimates from FLOPs and bytes; the speed stage measures the real ones.
python -m ccslm.estimate plan --scores out/prune/scores.json # drop 0..10 Mamba layers, bf16 and GPTQ
python -m ccslm.estimate model --model out/runs/A/merged
"""
from __future__ import annotations
import argparse
import json
import math
import re
from pathlib import Path
from typing import Any
from ccslm.common import MAMBA_TYPES, base_dir, definition, log, read_json
MAX_LEN = 122_880 + 3_072
BYTES = {"bf16": 2.0, "fp16": 2.0, "gptq": 0.5, "awq": 0.5}
R_CLASS = {"1b": 0.02, "3b": 0.05, "8b": 0.1}
FRONTIER = {"claude-haiku-5-5": (0.1, 0.5), "glm-5-3-flash": (0.048, 0.2)}
WEIGHTS = {"ttft": 0.4, "memory": 0.3, "parameters": 0.2, "tpot": 0.1}
LAYER = re.compile(r"^model\.layers\.(\d+)\.")
def layer_parameters(model_dir: Path) -> tuple[int, dict[int, int]]:
"""Parameters outside the layers and per layer index, from the safetensors headers."""
from microtensor_subnet.artifact.params import _counted, read_header, safetensors_in
outside = 0
per: dict[int, int] = {}
for path in safetensors_in(model_dir):
for name, entry in read_header(path).items():
if not isinstance(entry, dict):
continue
count = _counted(name, entry)
found = LAYER.match(name)
if found:
per[int(found.group(1))] = per.get(int(found.group(1)), 0) + count
else:
outside += count
return outside, per
def prices() -> tuple[dict[str, float], dict[str, tuple[float, float]]]:
try:
found = definition()
except Exception: # noqa: BLE001 - offline: the published values
return dict(R_CLASS), dict(FRONTIER)
r_class = {k: float(v) for k, v in (found.get("r_class") or R_CLASS).items()}
frontier = {
str(b["model"]): (float(b["usd_in_per_mtok"]), float(b["usd_out_per_mtok"]))
for b in found.get("baseline_models") or []
if "usd_in_per_mtok" in b
} or dict(FRONTIER)
return r_class, frontier
def token_price(parameters: int, r_class: dict[str, float]) -> float:
"""validator/context.py token_price: the class is chosen by the parameter count."""
if parameters <= 1_000_000_000:
return r_class.get("1b", 0.0)
if parameters <= 3_000_000_000:
return r_class.get("3b", 0.0)
return r_class.get("8b", 0.0)
def profile(config: dict[str, Any], outside: int, per: dict[int, int], *, drop: tuple[int, ...] = (), fmt: str = "bf16") -> dict[str, Any]:
kept = [i for i in sorted(per) if i not in set(drop)]
types = [config["layer_types"][i] for i in kept]
parameters = outside + sum(per[i] for i in kept)
heads = int(config["num_attention_heads"])
kv_heads = int(config.get("num_key_value_heads") or heads)
hidden = int(config["hidden_size"])
head_dim = int(config.get("head_dim") or hidden // heads)
kv_formula = 2 * len(kept) * kv_heads * head_dim * MAX_LEN * 2
attention_layers = sum(1 for t in types if t not in MAMBA_TYPES)
embedding = int(config["vocab_size"]) * hidden
return {
"layers": len(kept),
"dropped": sorted(drop),
"format": fmt,
"parameters": parameters,
"memory_bytes": int(parameters * BYTES[fmt]) + kv_formula,
"matmul_parameters": parameters - embedding,
"attention_layers": attention_layers,
"hidden": hidden,
"kv_per_token_bytes": attention_layers * 2 * kv_heads * head_dim * 2,
"weight_bytes": parameters * BYTES[fmt],
}
def prefill_flops(p: dict[str, Any], tokens: int) -> float:
# weights once per token, plus causal attention (QK and AV) in the attention layers
return 2.0 * p["matmul_parameters"] * tokens + 2.0 * p["attention_layers"] * tokens * tokens * p["hidden"]
def decode_bytes(p: dict[str, Any], context: int) -> float:
return p["weight_bytes"] + p["kv_per_token_bytes"] * context
def gains(mine: dict[str, Any], ref: dict[str, Any], tokens_in: int, tokens_out: int) -> dict[str, float]:
def gain(r: float, m: float) -> float:
return min(1.0, max(0.0, (r - m) / r)) if r > 0 else 0.0
found = {
"ttft": gain(prefill_flops(ref, tokens_in), prefill_flops(mine, tokens_in)),
"memory": gain(ref["memory_bytes"], mine["memory_bytes"]),
"parameters": gain(ref["parameters"], mine["parameters"]),
# a 4 bit kernel still dequantises, so decode gains less than the byte ratio; count half of it
"tpot": gain(decode_bytes(ref, tokens_in), decode_bytes(mine, tokens_in)) * (0.5 if mine["format"] in ("gptq", "awq") else 1.0),
}
if mine["format"] in ("gptq", "awq"):
found["ttft"] = min(found["ttft"], gain(prefill_flops(ref, tokens_in), prefill_flops({**mine, "format": "bf16"}, tokens_in)))
found["E"] = sum(WEIGHTS[k] * found[k] for k in WEIGHTS)
return found
def costs(parameters: int, tokens_in: int, tokens_out: int) -> dict[str, Any]:
r_class, frontier = prices()
price = token_price(parameters, r_class)
mine = (tokens_in + tokens_out) * price
row: dict[str, Any] = {"price_per_token_micros": price, "item_cost_micros": round(mine)}
for name, (usd_in, usd_out) in frontier.items():
theirs = tokens_in * usd_in + tokens_out * usd_out
row[f"vs_{name}"] = round(theirs / mine, 2) if mine else None
# the cost half of the system pool's v_f for this ratio (latency term left out)
row[f"v_cost_{name}"] = round(min(1.0, max(0.0, 0.7 * math.log(theirs / mine) / math.log(10))), 3) if mine else None
return row
def plan(model_dir: Path, scores: Path | None, max_drop: int, tokens_in: int, tokens_out: int) -> list[dict[str, Any]]:
from ccslm.prune import candidates
config = json.loads((model_dir / "config.json").read_text(encoding="utf-8"))
outside, per = layer_parameters(model_dir)
ranked = read_json(scores, {}).get("ranked") if scores else None
order = list(ranked or sorted(candidates(config), reverse=True))
ref = profile(config, outside, per)
rows = []
for k in range(0, min(max_drop, len(order)) + 1):
for fmt in ("bf16", "gptq"):
mine = profile(config, outside, per, drop=tuple(order[:k]), fmt=fmt)
found = gains(mine, ref, tokens_in, tokens_out)
rows.append(
{
"drop": k,
"format": fmt,
"parameters": mine["parameters"],
"memory_gb": round(mine["memory_bytes"] / 1e9, 2),
"E": round(found["E"], 3),
"slm_multiplier": round(1 + 0.25 * found["E"], 3),
**{f"e_{k2}": round(v, 3) for k2, v in found.items() if k2 != "E"},
**costs(mine["parameters"], tokens_in, tokens_out),
"layers_dropped": mine["dropped"],
}
)
return rows
def main() -> None:
parser = argparse.ArgumentParser(description="parameters, memory, E and cost by the arena's formulas")
sub = parser.add_subparsers(dest="cmd", required=True)
p_plan = sub.add_parser("plan", help="drop 0..N Mamba layers, bf16 and GPTQ")
p_plan.add_argument("--model", default="", help="default: the pinned base")
p_plan.add_argument("--scores", default="", help="prune.py scores.json for the drop order")
p_plan.add_argument("--max-drop", type=int, default=10)
p_model = sub.add_parser("model", help="one model folder against the base")
p_model.add_argument("--model", required=True)
for p in (p_plan, p_model):
p.add_argument("--input-tokens", type=int, default=60_000, help="a typical hidden item is 35k-95k")
p.add_argument("--output-tokens", type=int, default=720, help="compact targets average ~720")
p.add_argument("--json", action="store_true")
args = parser.parse_args()
if args.cmd == "plan":
rows = plan(Path(args.model) if args.model else base_dir(), Path(args.scores) if args.scores else None, args.max_drop, args.input_tokens, args.output_tokens)
if args.json:
print(json.dumps(rows, indent=2))
return
print(f"{'drop':>4} {'fmt':5} {'params':>13} {'mem GB':>7} {'E':>6} {'xSLM':>6} {'price':>6} {'vs Haiku':>8} {'vs GLM':>7}")
for r in rows:
print(
f"{r['drop']:>4} {r['format']:5} {r['parameters']:>13,} {r['memory_gb']:>7} {r['E']:>6} {r['slm_multiplier']:>6} "
f"{r['price_per_token_micros']:>6} {r['vs_claude-haiku-5-5']:>7}x {r['vs_glm-5-3-flash']:>6}x"
)
return
model = Path(args.model)
config = json.loads((model / "config.json").read_text(encoding="utf-8"))
quant = (config.get("quantization_config") or {}).get("quant_method", "")
base = base_dir()
base_config = json.loads((base / "config.json").read_text(encoding="utf-8"))
mine = profile(config, *layer_parameters(model), fmt=quant if quant in BYTES else "bf16")
ref = profile(base_config, *layer_parameters(base))
found = {"model": str(model), **mine, **{k: round(v, 4) for k, v in gains(mine, ref, args.input_tokens, args.output_tokens).items()}}
found.update(costs(mine["parameters"], args.input_tokens, args.output_tokens))
print(json.dumps(found, indent=2))
log(f"E ~ {found['E']}, SLM score x{1 + 0.25 * found['E']:.3f}, {found['price_per_token_micros']} micro-dollars per token")
if __name__ == "__main__":
main()