Miruamel's picture
Publish algo v1 (code-point tokenizer): 38.8M params, 16.4M tokens, loss 2.10
05f5056 verified
Raw History Blame Contribute Delete
2.62 kB
from __future__ import annotations
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from algo.config import ModelConfig, TrainingConfig
from algo.data import SimpleTokenizer, build_train_stream, synthetic_examples
from algo.eval import EvalScores, VerifierRepairLoop, strict_gate
from algo.rsi import RSIController
from algo.tools import build_envelope, execute_shell, execute_python, parse_envelope
try:
import torch
from algo.model import MiniTransformer
except ImportError as exc:
raise RuntimeError("torch is required for smoke; install project dependencies") from exc
def main() -> None:
cfg = ModelConfig()
assert cfg.parameter_count_tied == 38_806_016
assert cfg.parameter_count_untied == 47_194_624
tokenizer = SimpleTokenizer()
examples = synthetic_examples(count=2)
stream = list(build_train_stream(tokenizer, examples, sequence_length=8, max_tokens=16))
assert len(stream) == 2
assert all(len(batch) == 8 for batch in stream)
envelope = build_envelope("shell", {"ok": True, "output": "ok", "error": None, "exit_code": 0}, request_id="r1")
assert parse_envelope(envelope)["request_id"] == "r1"
assert execute_python("x = 2 + 3\nprint(x)").output.strip() == "5"
assert not execute_shell("rm -rf /tmp/blocked").ok
scores = EvalScores(math=0.72, coding=0.68, tool=0.86)
assert strict_gate(scores)
loop = VerifierRepairLoop(max_repairs=1)
attempts: list[str] = []
def verifier(answer: str) -> bool:
attempts.append(answer)
return len(attempts) == 2
repaired, ok = loop.run("bad", verifier, repair=lambda answer, _feedback: answer + " repaired")
assert ok
assert attempts == ["bad", "bad repaired"]
baseline = EvalScores(math=0.70, coding=0.65, tool=0.85)
controller = RSIController(baseline)
controller.apply(scores)
assert controller.accept()
controller.revert()
assert not controller.accept()
if torch is not None and MiniTransformer is not None:
small = ModelConfig(num_layers=1, d_model=32, num_query_heads=4, num_kv_heads=2, ffn_dim=64, vocab_size=128, context_length=16)
model = MiniTransformer(small)
with torch.no_grad():
logits = model(torch.randint(0, small.vocab_size, (1, 8)))
assert tuple(logits.shape) == (1, 8, small.vocab_size)
assert model.parameter_count == small.parameter_count_tied
print(json.dumps({"status": "ok", "parameter_count_tied": cfg.parameter_count_tied}, sort_keys=True))
if __name__ == "__main__":
main()