algo-v1-codepoint / tests /test_e2e.py
Miruamel's picture
Publish algo v1 (code-point tokenizer): 38.8M params, 16.4M tokens, loss 2.10
05f5056 verified
Raw History Blame Contribute Delete
8.65 kB
from __future__ import annotations
from pathlib import Path
from typing import Any, Mapping, Sequence
import pytest
import torch
from algo.config import ModelConfig, TrainingConfig
from algo.data import DatasetManifest, SimpleTokenizer
from algo.eval import EvalScores, evaluate_model, strict_gate
from algo.model import MiniTransformer
from algo.rsi import accept_experiment, run_rsi_cycle
from algo.tools import execute_python, execute_shell, execute_tool_call
from algo.train import load_checkpoint, save_checkpoint, set_seed, train_model
class _OracleTokenizer:
def __init__(self) -> None:
self._tokenizer = SimpleTokenizer(vocab_size=256)
self.eos_id = self._tokenizer.eos_id
def encode(self, text: str, add_eos: bool = True) -> list[int]:
return self._tokenizer.encode(text, add_eos=add_eos)
def decode(self, ids: Sequence[int]) -> str:
return self._tokenizer.decode(ids)
class _OracleModel:
training = False
def __init__(self, tokenizer: _OracleTokenizer, answers: Mapping[str, str]) -> None:
self.tokenizer = tokenizer
self.answers = answers
def to(self, device: str) -> "_OracleModel":
return self
def eval(self) -> "_OracleModel":
self.training = False
return self
def __call__(self, input_ids: torch.Tensor) -> torch.Tensor:
return torch.zeros(input_ids.shape[0], input_ids.shape[1], 256)
def generate(
self,
input_ids: torch.Tensor,
max_new_tokens: int = 64,
eos_id: int | None = None,
**_kwargs: Any,
) -> torch.Tensor:
prompt = self.tokenizer.decode(input_ids[0].tolist())
answer = self.answers[prompt]
answer_ids = self.tokenizer.encode(answer, add_eos=False)[:max_new_tokens]
suffix = torch.tensor([answer_ids], dtype=input_ids.dtype, device=input_ids.device)
return torch.cat((input_ids, suffix), dim=-1)
def test_manifest_plan_keys() -> None:
manifest = DatasetManifest.load("data/manifest.json")
assert manifest.sources["gsm8k"].commit == "740312add88f781978c0658806c59bc2815b9866"
assert manifest.sources["humaneval"].split == "test"
assert manifest.sources["tinystories"].license == "CDLA-sharing-1.0"
def test_sandbox_blocks_escape_and_format_attack() -> None:
assert execute_python("print(2+3)").output.strip() == "5"
assert not execute_python("print('{0.__class__}'.format(1))").ok
assert not execute_shell("cat /etc/passwd").ok
assert not execute_shell("grep x /etc/passwd").ok
def test_tool_call_rejects_bad_boundaries() -> None:
assert not execute_tool_call({"kind": "python"}).ok
assert not execute_tool_call({"kind": "python", "payload": {"code": 1}}).ok
assert not execute_tool_call({"kind": "shell", "payload": {"command": 1}}).ok
assert not execute_tool_call({"kind": "shell", "payload": {"command": "echo ok"}}, workspace="missing-workspace").ok
def test_evaluate_model_rejects_invalid_contracts() -> None:
with pytest.raises(ValueError, match="model must be callable"):
evaluate_model(object(), SimpleTokenizer(), [])
with pytest.raises(ValueError, match="tokenizer must define encode and decode"):
evaluate_model(_OracleModel(_OracleTokenizer(), {}), object(), [])
def test_tiny_model_generate_memory_eval_and_checkpoint(tmp_path: Path) -> None:
cfg = ModelConfig(num_layers=1, d_model=32, num_query_heads=4, num_kv_heads=2, ffn_dim=64, vocab_size=128, context_length=16)
model = MiniTransformer(cfg)
tokenizer = type("Tokenizer", (), {"encode": lambda self, text: [1, 2, 3]})()
input_ids = torch.tensor([[1, 2, 3]])
with torch.no_grad():
logits = model(input_ids)
assert tuple(logits.shape) == (1, 3, cfg.vocab_size)
generated = model.generate(input_ids, max_new_tokens=2)
assert tuple(generated.shape) == (1, 5)
padded = model.generate(input_ids, max_new_tokens=2, attention_mask=torch.tensor([[0, 0, 0, 1, 1]]))
assert tuple(padded.shape) == (1, 5)
mask_3d = torch.tensor([[[0, 1, 1]]], dtype=torch.bool)
generated_3d = model.generate(input_ids, max_new_tokens=2, attention_mask=mask_3d)
assert tuple(generated_3d.shape) == (1, 5)
short_mask = torch.tensor([[0, 1]], dtype=torch.bool)
generated_short = model.generate(input_ids, max_new_tokens=2, attention_mask=short_mask)
assert tuple(generated_short.shape) == (1, 5)
with pytest.raises(ValueError, match="batch size"):
model.generate(input_ids, attention_mask=torch.zeros(2, 3, dtype=torch.bool))
with pytest.raises(ValueError, match="shape"):
model.generate(input_ids, attention_mask=torch.zeros(1, 2, 3, dtype=torch.bool))
assert model.estimate_inference_memory_bytes() <= 500 * 1024 * 1024
examples = [
{"domain": "math", "text": "1+1", "answer": "2"},
{"domain": "coding", "text": "print(1)", "answer": "print(1)"},
{"domain": "tool", "text": "tool", "tool_call": {"kind": "python", "code": "print(1)"}},
]
oracle_tokenizer = _OracleTokenizer()
oracle_model = _OracleModel(oracle_tokenizer, {"1+1": "2", "print(1)": "print(1)"})
scores = evaluate_model(oracle_model, oracle_tokenizer, examples, tool_executor=lambda _kind, _payload: {"ok": True})
assert strict_gate(scores)
path = tmp_path / "state.pt"
state = {"step": 2, "tokens": 99}
save_checkpoint(path, state)
assert load_checkpoint(path) == state
def test_train_model_resume_matches_continuous_run(tmp_path: Path) -> None:
model_config = ModelConfig(
num_layers=1,
d_model=8,
num_query_heads=2,
num_kv_heads=1,
ffn_dim=16,
vocab_size=16,
context_length=8,
dropout=0.1,
)
config = TrainingConfig(
sequence_length=4,
global_batch_tokens=4,
max_tokens=8,
checkpoint_every_tokens=4,
warmup_steps=1,
)
tokenizer = SimpleTokenizer(vocab_size=model_config.vocab_size)
examples = [{"text": "abcd"}]
first_path = tmp_path / "first.pt"
full_path = tmp_path / "full.pt"
resume_path = tmp_path / "resume.pt"
set_seed(config.seed)
first_model = MiniTransformer(model_config)
first_result = train_model(
first_model,
tokenizer,
config,
examples=examples,
max_steps=1,
checkpoint_path=first_path,
)
assert first_result["step"] == 1
assert first_result["tokens_seen"] == 4
set_seed(config.seed)
full_model = MiniTransformer(model_config)
full_result = train_model(
full_model,
tokenizer,
config,
examples=examples,
max_steps=2,
checkpoint_path=full_path,
)
set_seed(config.seed)
resumed_model = MiniTransformer(model_config)
resumed_result = train_model(
resumed_model,
tokenizer,
config,
examples=examples,
resume_path=first_path,
max_steps=2,
checkpoint_path=resume_path,
)
assert full_result["step"] == 2
assert full_result["tokens_seen"] == 8
assert resumed_result == full_result
full_state = load_checkpoint(full_path)
resumed_state = load_checkpoint(resume_path)
assert full_state["step"] == resumed_state["step"] == 2
assert full_state["tokens_seen"] == resumed_state["tokens_seen"] == 8
for name, value in full_state["model"].items():
assert torch.equal(value, resumed_state["model"][name])
full_optimizer = full_state["optimizer"]
resumed_optimizer = resumed_state["optimizer"]
assert full_optimizer["param_groups"] == resumed_optimizer["param_groups"]
assert full_optimizer["state"].keys() == resumed_optimizer["state"].keys()
for parameter_id, full_parameter_state in full_optimizer["state"].items():
resumed_parameter_state = resumed_optimizer["state"][parameter_id]
assert full_parameter_state.keys() == resumed_parameter_state.keys()
for state_name, value in full_parameter_state.items():
assert torch.equal(value, resumed_parameter_state[state_name])
def test_rsi_accept_and_cycle() -> None:
baseline = EvalScores(math=0.70, coding=0.65, tool=0.85)
candidate = EvalScores(math=0.73, coding=0.67, tool=0.87)
assert accept_experiment(baseline, candidate).accepted
calls: list[str] = []
result = run_rsi_cycle(
baseline,
candidate,
apply_candidate=lambda: calls.append("apply"),
evaluate=lambda: candidate,
revert_candidate=lambda: calls.append("revert"),
)
assert result.accepted
assert calls == ["apply", "revert"]