from __future__ import annotations from pathlib import Path from typing import Any, Mapping, Sequence import pytest import torch from algo.config import ModelConfig, TrainingConfig from algo.data import DatasetManifest, SimpleTokenizer from algo.eval import EvalScores, evaluate_model, strict_gate from algo.model import MiniTransformer from algo.rsi import accept_experiment, run_rsi_cycle from algo.tools import execute_python, execute_shell, execute_tool_call from algo.train import load_checkpoint, save_checkpoint, set_seed, train_model class _OracleTokenizer: def __init__(self) -> None: self._tokenizer = SimpleTokenizer(vocab_size=256) self.eos_id = self._tokenizer.eos_id def encode(self, text: str, add_eos: bool = True) -> list[int]: return self._tokenizer.encode(text, add_eos=add_eos) def decode(self, ids: Sequence[int]) -> str: return self._tokenizer.decode(ids) class _OracleModel: training = False def __init__(self, tokenizer: _OracleTokenizer, answers: Mapping[str, str]) -> None: self.tokenizer = tokenizer self.answers = answers def to(self, device: str) -> "_OracleModel": return self def eval(self) -> "_OracleModel": self.training = False return self def __call__(self, input_ids: torch.Tensor) -> torch.Tensor: return torch.zeros(input_ids.shape[0], input_ids.shape[1], 256) def generate( self, input_ids: torch.Tensor, max_new_tokens: int = 64, eos_id: int | None = None, **_kwargs: Any, ) -> torch.Tensor: prompt = self.tokenizer.decode(input_ids[0].tolist()) answer = self.answers[prompt] answer_ids = self.tokenizer.encode(answer, add_eos=False)[:max_new_tokens] suffix = torch.tensor([answer_ids], dtype=input_ids.dtype, device=input_ids.device) return torch.cat((input_ids, suffix), dim=-1) def test_manifest_plan_keys() -> None: manifest = DatasetManifest.load("data/manifest.json") assert manifest.sources["gsm8k"].commit == "740312add88f781978c0658806c59bc2815b9866" assert manifest.sources["humaneval"].split == "test" assert manifest.sources["tinystories"].license == "CDLA-sharing-1.0" def test_sandbox_blocks_escape_and_format_attack() -> None: assert execute_python("print(2+3)").output.strip() == "5" assert not execute_python("print('{0.__class__}'.format(1))").ok assert not execute_shell("cat /etc/passwd").ok assert not execute_shell("grep x /etc/passwd").ok def test_tool_call_rejects_bad_boundaries() -> None: assert not execute_tool_call({"kind": "python"}).ok assert not execute_tool_call({"kind": "python", "payload": {"code": 1}}).ok assert not execute_tool_call({"kind": "shell", "payload": {"command": 1}}).ok assert not execute_tool_call({"kind": "shell", "payload": {"command": "echo ok"}}, workspace="missing-workspace").ok def test_evaluate_model_rejects_invalid_contracts() -> None: with pytest.raises(ValueError, match="model must be callable"): evaluate_model(object(), SimpleTokenizer(), []) with pytest.raises(ValueError, match="tokenizer must define encode and decode"): evaluate_model(_OracleModel(_OracleTokenizer(), {}), object(), []) def test_tiny_model_generate_memory_eval_and_checkpoint(tmp_path: Path) -> None: cfg = ModelConfig(num_layers=1, d_model=32, num_query_heads=4, num_kv_heads=2, ffn_dim=64, vocab_size=128, context_length=16) model = MiniTransformer(cfg) tokenizer = type("Tokenizer", (), {"encode": lambda self, text: [1, 2, 3]})() input_ids = torch.tensor([[1, 2, 3]]) with torch.no_grad(): logits = model(input_ids) assert tuple(logits.shape) == (1, 3, cfg.vocab_size) generated = model.generate(input_ids, max_new_tokens=2) assert tuple(generated.shape) == (1, 5) padded = model.generate(input_ids, max_new_tokens=2, attention_mask=torch.tensor([[0, 0, 0, 1, 1]])) assert tuple(padded.shape) == (1, 5) mask_3d = torch.tensor([[[0, 1, 1]]], dtype=torch.bool) generated_3d = model.generate(input_ids, max_new_tokens=2, attention_mask=mask_3d) assert tuple(generated_3d.shape) == (1, 5) short_mask = torch.tensor([[0, 1]], dtype=torch.bool) generated_short = model.generate(input_ids, max_new_tokens=2, attention_mask=short_mask) assert tuple(generated_short.shape) == (1, 5) with pytest.raises(ValueError, match="batch size"): model.generate(input_ids, attention_mask=torch.zeros(2, 3, dtype=torch.bool)) with pytest.raises(ValueError, match="shape"): model.generate(input_ids, attention_mask=torch.zeros(1, 2, 3, dtype=torch.bool)) assert model.estimate_inference_memory_bytes() <= 500 * 1024 * 1024 examples = [ {"domain": "math", "text": "1+1", "answer": "2"}, {"domain": "coding", "text": "print(1)", "answer": "print(1)"}, {"domain": "tool", "text": "tool", "tool_call": {"kind": "python", "code": "print(1)"}}, ] oracle_tokenizer = _OracleTokenizer() oracle_model = _OracleModel(oracle_tokenizer, {"1+1": "2", "print(1)": "print(1)"}) scores = evaluate_model(oracle_model, oracle_tokenizer, examples, tool_executor=lambda _kind, _payload: {"ok": True}) assert strict_gate(scores) path = tmp_path / "state.pt" state = {"step": 2, "tokens": 99} save_checkpoint(path, state) assert load_checkpoint(path) == state def test_train_model_resume_matches_continuous_run(tmp_path: Path) -> None: model_config = ModelConfig( num_layers=1, d_model=8, num_query_heads=2, num_kv_heads=1, ffn_dim=16, vocab_size=16, context_length=8, dropout=0.1, ) config = TrainingConfig( sequence_length=4, global_batch_tokens=4, max_tokens=8, checkpoint_every_tokens=4, warmup_steps=1, ) tokenizer = SimpleTokenizer(vocab_size=model_config.vocab_size) examples = [{"text": "abcd"}] first_path = tmp_path / "first.pt" full_path = tmp_path / "full.pt" resume_path = tmp_path / "resume.pt" set_seed(config.seed) first_model = MiniTransformer(model_config) first_result = train_model( first_model, tokenizer, config, examples=examples, max_steps=1, checkpoint_path=first_path, ) assert first_result["step"] == 1 assert first_result["tokens_seen"] == 4 set_seed(config.seed) full_model = MiniTransformer(model_config) full_result = train_model( full_model, tokenizer, config, examples=examples, max_steps=2, checkpoint_path=full_path, ) set_seed(config.seed) resumed_model = MiniTransformer(model_config) resumed_result = train_model( resumed_model, tokenizer, config, examples=examples, resume_path=first_path, max_steps=2, checkpoint_path=resume_path, ) assert full_result["step"] == 2 assert full_result["tokens_seen"] == 8 assert resumed_result == full_result full_state = load_checkpoint(full_path) resumed_state = load_checkpoint(resume_path) assert full_state["step"] == resumed_state["step"] == 2 assert full_state["tokens_seen"] == resumed_state["tokens_seen"] == 8 for name, value in full_state["model"].items(): assert torch.equal(value, resumed_state["model"][name]) full_optimizer = full_state["optimizer"] resumed_optimizer = resumed_state["optimizer"] assert full_optimizer["param_groups"] == resumed_optimizer["param_groups"] assert full_optimizer["state"].keys() == resumed_optimizer["state"].keys() for parameter_id, full_parameter_state in full_optimizer["state"].items(): resumed_parameter_state = resumed_optimizer["state"][parameter_id] assert full_parameter_state.keys() == resumed_parameter_state.keys() for state_name, value in full_parameter_state.items(): assert torch.equal(value, resumed_parameter_state[state_name]) def test_rsi_accept_and_cycle() -> None: baseline = EvalScores(math=0.70, coding=0.65, tool=0.85) candidate = EvalScores(math=0.73, coding=0.67, tool=0.87) assert accept_experiment(baseline, candidate).accepted calls: list[str] = [] result = run_rsi_cycle( baseline, candidate, apply_candidate=lambda: calls.append("apply"), evaluate=lambda: candidate, revert_candidate=lambda: calls.append("revert"), ) assert result.accepted assert calls == ["apply", "revert"]