Download tests/test_e2e.py from Miruamel/algo-v1-codepoint: direct link, hf CLI and curl.
- Browser
- Download file 8.65 kB
-
https://huggingface.co/Miruamel/algo-v1-codepoint/resolve/main/tests/test_e2e.py
- Command line
-
hf download hf://Miruamel/algo-v1-codepoint/tests/test_e2e.py
-
curl -L -o test_e2e.py https://huggingface.co/Miruamel/algo-v1-codepoint/resolve/main/tests/test_e2e.py
8.65 kB
| from __future__ import annotations | |
| from pathlib import Path | |
| from typing import Any, Mapping, Sequence | |
| import pytest | |
| import torch | |
| from algo.config import ModelConfig, TrainingConfig | |
| from algo.data import DatasetManifest, SimpleTokenizer | |
| from algo.eval import EvalScores, evaluate_model, strict_gate | |
| from algo.model import MiniTransformer | |
| from algo.rsi import accept_experiment, run_rsi_cycle | |
| from algo.tools import execute_python, execute_shell, execute_tool_call | |
| from algo.train import load_checkpoint, save_checkpoint, set_seed, train_model | |
| class _OracleTokenizer: | |
| def __init__(self) -> None: | |
| self._tokenizer = SimpleTokenizer(vocab_size=256) | |
| self.eos_id = self._tokenizer.eos_id | |
| def encode(self, text: str, add_eos: bool = True) -> list[int]: | |
| return self._tokenizer.encode(text, add_eos=add_eos) | |
| def decode(self, ids: Sequence[int]) -> str: | |
| return self._tokenizer.decode(ids) | |
| class _OracleModel: | |
| training = False | |
| def __init__(self, tokenizer: _OracleTokenizer, answers: Mapping[str, str]) -> None: | |
| self.tokenizer = tokenizer | |
| self.answers = answers | |
| def to(self, device: str) -> "_OracleModel": | |
| return self | |
| def eval(self) -> "_OracleModel": | |
| self.training = False | |
| return self | |
| def __call__(self, input_ids: torch.Tensor) -> torch.Tensor: | |
| return torch.zeros(input_ids.shape[0], input_ids.shape[1], 256) | |
| def generate( | |
| self, | |
| input_ids: torch.Tensor, | |
| max_new_tokens: int = 64, | |
| eos_id: int | None = None, | |
| **_kwargs: Any, | |
| ) -> torch.Tensor: | |
| prompt = self.tokenizer.decode(input_ids[0].tolist()) | |
| answer = self.answers[prompt] | |
| answer_ids = self.tokenizer.encode(answer, add_eos=False)[:max_new_tokens] | |
| suffix = torch.tensor([answer_ids], dtype=input_ids.dtype, device=input_ids.device) | |
| return torch.cat((input_ids, suffix), dim=-1) | |
| def test_manifest_plan_keys() -> None: | |
| manifest = DatasetManifest.load("data/manifest.json") | |
| assert manifest.sources["gsm8k"].commit == "740312add88f781978c0658806c59bc2815b9866" | |
| assert manifest.sources["humaneval"].split == "test" | |
| assert manifest.sources["tinystories"].license == "CDLA-sharing-1.0" | |
| def test_sandbox_blocks_escape_and_format_attack() -> None: | |
| assert execute_python("print(2+3)").output.strip() == "5" | |
| assert not execute_python("print('{0.__class__}'.format(1))").ok | |
| assert not execute_shell("cat /etc/passwd").ok | |
| assert not execute_shell("grep x /etc/passwd").ok | |
| def test_tool_call_rejects_bad_boundaries() -> None: | |
| assert not execute_tool_call({"kind": "python"}).ok | |
| assert not execute_tool_call({"kind": "python", "payload": {"code": 1}}).ok | |
| assert not execute_tool_call({"kind": "shell", "payload": {"command": 1}}).ok | |
| assert not execute_tool_call({"kind": "shell", "payload": {"command": "echo ok"}}, workspace="missing-workspace").ok | |
| def test_evaluate_model_rejects_invalid_contracts() -> None: | |
| with pytest.raises(ValueError, match="model must be callable"): | |
| evaluate_model(object(), SimpleTokenizer(), []) | |
| with pytest.raises(ValueError, match="tokenizer must define encode and decode"): | |
| evaluate_model(_OracleModel(_OracleTokenizer(), {}), object(), []) | |
| def test_tiny_model_generate_memory_eval_and_checkpoint(tmp_path: Path) -> None: | |
| cfg = ModelConfig(num_layers=1, d_model=32, num_query_heads=4, num_kv_heads=2, ffn_dim=64, vocab_size=128, context_length=16) | |
| model = MiniTransformer(cfg) | |
| tokenizer = type("Tokenizer", (), {"encode": lambda self, text: [1, 2, 3]})() | |
| input_ids = torch.tensor([[1, 2, 3]]) | |
| with torch.no_grad(): | |
| logits = model(input_ids) | |
| assert tuple(logits.shape) == (1, 3, cfg.vocab_size) | |
| generated = model.generate(input_ids, max_new_tokens=2) | |
| assert tuple(generated.shape) == (1, 5) | |
| padded = model.generate(input_ids, max_new_tokens=2, attention_mask=torch.tensor([[0, 0, 0, 1, 1]])) | |
| assert tuple(padded.shape) == (1, 5) | |
| mask_3d = torch.tensor([[[0, 1, 1]]], dtype=torch.bool) | |
| generated_3d = model.generate(input_ids, max_new_tokens=2, attention_mask=mask_3d) | |
| assert tuple(generated_3d.shape) == (1, 5) | |
| short_mask = torch.tensor([[0, 1]], dtype=torch.bool) | |
| generated_short = model.generate(input_ids, max_new_tokens=2, attention_mask=short_mask) | |
| assert tuple(generated_short.shape) == (1, 5) | |
| with pytest.raises(ValueError, match="batch size"): | |
| model.generate(input_ids, attention_mask=torch.zeros(2, 3, dtype=torch.bool)) | |
| with pytest.raises(ValueError, match="shape"): | |
| model.generate(input_ids, attention_mask=torch.zeros(1, 2, 3, dtype=torch.bool)) | |
| assert model.estimate_inference_memory_bytes() <= 500 * 1024 * 1024 | |
| examples = [ | |
| {"domain": "math", "text": "1+1", "answer": "2"}, | |
| {"domain": "coding", "text": "print(1)", "answer": "print(1)"}, | |
| {"domain": "tool", "text": "tool", "tool_call": {"kind": "python", "code": "print(1)"}}, | |
| ] | |
| oracle_tokenizer = _OracleTokenizer() | |
| oracle_model = _OracleModel(oracle_tokenizer, {"1+1": "2", "print(1)": "print(1)"}) | |
| scores = evaluate_model(oracle_model, oracle_tokenizer, examples, tool_executor=lambda _kind, _payload: {"ok": True}) | |
| assert strict_gate(scores) | |
| path = tmp_path / "state.pt" | |
| state = {"step": 2, "tokens": 99} | |
| save_checkpoint(path, state) | |
| assert load_checkpoint(path) == state | |
| def test_train_model_resume_matches_continuous_run(tmp_path: Path) -> None: | |
| model_config = ModelConfig( | |
| num_layers=1, | |
| d_model=8, | |
| num_query_heads=2, | |
| num_kv_heads=1, | |
| ffn_dim=16, | |
| vocab_size=16, | |
| context_length=8, | |
| dropout=0.1, | |
| ) | |
| config = TrainingConfig( | |
| sequence_length=4, | |
| global_batch_tokens=4, | |
| max_tokens=8, | |
| checkpoint_every_tokens=4, | |
| warmup_steps=1, | |
| ) | |
| tokenizer = SimpleTokenizer(vocab_size=model_config.vocab_size) | |
| examples = [{"text": "abcd"}] | |
| first_path = tmp_path / "first.pt" | |
| full_path = tmp_path / "full.pt" | |
| resume_path = tmp_path / "resume.pt" | |
| set_seed(config.seed) | |
| first_model = MiniTransformer(model_config) | |
| first_result = train_model( | |
| first_model, | |
| tokenizer, | |
| config, | |
| examples=examples, | |
| max_steps=1, | |
| checkpoint_path=first_path, | |
| ) | |
| assert first_result["step"] == 1 | |
| assert first_result["tokens_seen"] == 4 | |
| set_seed(config.seed) | |
| full_model = MiniTransformer(model_config) | |
| full_result = train_model( | |
| full_model, | |
| tokenizer, | |
| config, | |
| examples=examples, | |
| max_steps=2, | |
| checkpoint_path=full_path, | |
| ) | |
| set_seed(config.seed) | |
| resumed_model = MiniTransformer(model_config) | |
| resumed_result = train_model( | |
| resumed_model, | |
| tokenizer, | |
| config, | |
| examples=examples, | |
| resume_path=first_path, | |
| max_steps=2, | |
| checkpoint_path=resume_path, | |
| ) | |
| assert full_result["step"] == 2 | |
| assert full_result["tokens_seen"] == 8 | |
| assert resumed_result == full_result | |
| full_state = load_checkpoint(full_path) | |
| resumed_state = load_checkpoint(resume_path) | |
| assert full_state["step"] == resumed_state["step"] == 2 | |
| assert full_state["tokens_seen"] == resumed_state["tokens_seen"] == 8 | |
| for name, value in full_state["model"].items(): | |
| assert torch.equal(value, resumed_state["model"][name]) | |
| full_optimizer = full_state["optimizer"] | |
| resumed_optimizer = resumed_state["optimizer"] | |
| assert full_optimizer["param_groups"] == resumed_optimizer["param_groups"] | |
| assert full_optimizer["state"].keys() == resumed_optimizer["state"].keys() | |
| for parameter_id, full_parameter_state in full_optimizer["state"].items(): | |
| resumed_parameter_state = resumed_optimizer["state"][parameter_id] | |
| assert full_parameter_state.keys() == resumed_parameter_state.keys() | |
| for state_name, value in full_parameter_state.items(): | |
| assert torch.equal(value, resumed_parameter_state[state_name]) | |
| def test_rsi_accept_and_cycle() -> None: | |
| baseline = EvalScores(math=0.70, coding=0.65, tool=0.85) | |
| candidate = EvalScores(math=0.73, coding=0.67, tool=0.87) | |
| assert accept_experiment(baseline, candidate).accepted | |
| calls: list[str] = [] | |
| result = run_rsi_cycle( | |
| baseline, | |
| candidate, | |
| apply_candidate=lambda: calls.append("apply"), | |
| evaluate=lambda: candidate, | |
| revert_candidate=lambda: calls.append("revert"), | |
| ) | |
| assert result.accepted | |
| assert calls == ["apply", "revert"] | |