Spaces:
Running
Running
| """Extensions unique_ext (v3.9, ADR-002) : non-régression stricte, briques d'inférence, phases d'entraînement, jobs. | |
| Règle vérifiée en premier : sans aucune variable UNIQUE_EXT_* / UNIQUE_PHASES, rien ne change (menu, job, panneau). | |
| """ | |
| from __future__ import annotations | |
| import importlib | |
| import importlib.util | |
| import json | |
| import os | |
| import sys | |
| from pathlib import Path | |
| from types import SimpleNamespace | |
| import pytest | |
| from mouh.learning.prompts import chat_messages, render_files | |
| from mouh.learning.records import split_for, task_hash | |
| ROOT = Path(__file__).resolve().parents[1] | |
| FENCE = "`" * 3 | |
| TESTS = {"test_solution.py": "from solution import add\n\n\ndef test_add():\n assert add(2, 3) == 5\n\n\n" | |
| "def test_zero():\n assert add(0, 0) == 0\n"} | |
| V38_CHOICES = ("Vérification seule (code fourni)", "Modèle M.O.U.H.", "UNIQUE (modèle entraîné)", | |
| "UNIQUE réfléchi (effort adapté à la difficulté)") | |
| def no_ext_env(monkeypatch): | |
| """Chaque test part d'un Space sans aucune extension activée.""" | |
| for k in list(os.environ): | |
| if k.startswith("UNIQUE_EXT_") or k == "UNIQUE_PHASES": | |
| monkeypatch.delenv(k) | |
| def load(path: str, name: str): | |
| spec = importlib.util.spec_from_file_location(name, ROOT / path) | |
| mod = importlib.util.module_from_spec(spec) | |
| spec.loader.exec_module(mod) | |
| return mod | |
| def files(body: str) -> str: | |
| return f"### FILE: solution.py\n{FENCE}python\n{body}\n{FENCE}\n### END" | |
| def request(tests=TESTS, attempt=1, instruction="additionner a et b"): | |
| return SimpleNamespace(task=SimpleNamespace(instruction=instruction, test_files=tests), attempt=attempt, | |
| previous_files=None, diagnostics=()) | |
| def build_for(instruction, tests): | |
| def build(previous=None, diagnostics=None): | |
| return chat_messages(instruction, tests, previous, diagnostics if previous else None) | |
| return build | |
| class ScriptedBackend: | |
| """Propositions fausses tant qu'on ne lui montre pas le code qui échoue ; la réparation est juste.""" | |
| name = "scripted" | |
| def __init__(self, first="return a - b", repair="return a + b"): | |
| self.first, self.repair, self.calls, self.generations = first, repair, [], 0 | |
| def complete(self, messages, n=1, *, greedy_first=True, max_new_tokens=None): | |
| self.calls.append(messages[-1]["content"]) | |
| self.generations += n | |
| fixing = "Code actuel (échoue)" in messages[-1]["content"] | |
| out = [] | |
| for i in range(n): | |
| body = self.repair if fixing else self.first | |
| out.append(files(f"def add(a, b):\n {body} # {len(self.calls)}.{i}")) | |
| return out | |
| # ================================================================= non-régression | |
| def test_nothing_changes_without_variables(): | |
| from unique_ext import registry | |
| assert registry.generator_choices() == () | |
| assert registry.phases() == [] | |
| assert registry.job_setup(lambda m: None) == [] | |
| def test_menu_v38_intact_and_extension_appended(monkeypatch): | |
| import mouh.service as S | |
| assert S.GENERATOR_CHOICES[:4] == V38_CHOICES and S.EXT_CHOICES == () | |
| monkeypatch.setenv("UNIQUE_EXT_TREE", "1") | |
| S2 = importlib.reload(S) | |
| try: | |
| assert S2.GENERATOR_CHOICES[:4] == V38_CHOICES | |
| assert S2.EXT_CHOICES == ("UNIQUE · recherche arborescente (MCTS + pytest)",) | |
| assert S2.GENERATOR_CHOICES[4:] == S2.EXT_CHOICES | |
| finally: | |
| monkeypatch.delenv("UNIQUE_EXT_TREE") | |
| importlib.reload(S) | |
| def test_broken_package_falls_back_to_v38(monkeypatch): | |
| import mouh.service as S | |
| monkeypatch.setitem(sys.modules, "unique_ext.registry", None) # import impossible | |
| S2 = importlib.reload(S) | |
| try: | |
| assert S2.EXT_CHOICES == () and S2.GENERATOR_CHOICES == V38_CHOICES | |
| finally: | |
| monkeypatch.delitem(sys.modules, "unique_ext.registry") | |
| importlib.reload(S) | |
| def test_phases_parsing_ignores_unknown_names(): | |
| from unique_ext.registry import phases | |
| assert phases("dpo:200, inconnue, KTO, dpo:x") == [("dpo", 200), ("kto", 100)] | |
| assert phases("") == [] | |
| def test_training_job_ignores_extensions_without_variables(monkeypatch): | |
| job = load("jobs/train_unique.py", "train_unique_ext_t1") | |
| assert job.load_extensions(None) is None | |
| monkeypatch.setenv("UNIQUE_PHASES", "dpo:16") | |
| monkeypatch.setenv("UNIQUE_EXT_PATH", str(ROOT)) | |
| reg = job.load_extensions(None) | |
| assert reg is not None and reg.phases() == [("dpo", 16)] | |
| import unique_ext | |
| monkeypatch.setattr(unique_ext, "API_VERSION", 99) # paquet d'une autre version : le job l'ignore | |
| assert job.load_extensions(None) is None | |
| # ================================================================= bus JEV : compilation JIT + cache | |
| def test_compile_gate_rejects_syntax_errors_without_running_pytest(): | |
| from unique_ext.jev.compile_gate import VerificationCache, compile_files | |
| from unique_ext.reasoning.verify import PytestVerifier | |
| rep = compile_files({"solution.py": "def add(a, b)\n return a + b\n"}) | |
| assert not rep.ok and "solution.py:1" in rep.errors[0] and "SyntaxError" in rep.errors[0] | |
| assert compile_files({"solution.py": "def add(a, b):\n return a + b\n"}).bytecode_bytes > 0 | |
| v = PytestVerifier(cache=VerificationCache()) | |
| c = v.score({"solution.py": "def add(a, b)\n return a + b\n"}, TESTS) | |
| assert not c.ok and v.runs == 0 and v.compile_rejects == 1 and "Compilation refusée" in c.output | |
| good = {"solution.py": "def add(a, b):\n return a + b\n"} | |
| assert v.score(good, TESTS).ok and v.runs == 1 | |
| again = v.score(good, TESTS) | |
| assert again.ok and again.cached and v.runs == 1 # même couple (fichiers, tests) : pas de second pytest | |
| def test_verifier_counts_partial_success(): | |
| from unique_ext.reasoning.verify import PytestVerifier | |
| c = PytestVerifier(cache=None).score({"solution.py": "def add(a, b):\n return a * b\n"}, TESTS) | |
| assert not c.ok and c.passed == 1 and c.failed == 1 and c.value == 0.5 | |
| # ================================================================= TTC : best-of-N, faisceau, MCTS | |
| def test_mcts_repairs_with_the_real_pytest_trace_and_respects_budget(): | |
| from unique_ext.contracts import Plan | |
| from unique_ext.reasoning.tree import RepairMCTS | |
| from unique_ext.reasoning.verify import PytestVerifier | |
| be = ScriptedBackend() | |
| cands = RepairMCTS().solve(request(), Plan("complexe", "mcts", budget=5), be, PytestVerifier(cache=None), | |
| build_for("additionner a et b", TESTS)) | |
| assert cands[0].ok and cands[0].origin == "reparation" and cands[0].depth == 1 | |
| assert be.generations <= 5 | |
| assert any("assert" in c for c in be.calls if "Code actuel" in c) # la trace pytest est montrée au modèle | |
| def test_beam_repairs_and_best_of_n_spends_whole_budget(): | |
| from unique_ext.contracts import Plan | |
| from unique_ext.reasoning.tree import BestOfN, RepairBeam | |
| from unique_ext.reasoning.verify import PytestVerifier | |
| be = ScriptedBackend() | |
| best = RepairBeam(width=1, depth=1).solve(request(), Plan("complexe", "arbre", budget=3), be, | |
| PytestVerifier(cache=None), build_for("x", TESTS))[0] | |
| assert best.ok and best.origin == "reparation" | |
| be2 = ScriptedBackend() | |
| out = BestOfN().solve(request(), Plan("moyenne", "direct", budget=3), be2, PytestVerifier(cache=None), | |
| build_for("x", TESTS)) | |
| assert be2.generations == 3 and not out[0].ok # sans réparation, le modèle scripté ne trouve pas | |
| def test_mcts_stops_at_budget_when_nothing_parses(): | |
| from unique_ext.contracts import Plan | |
| from unique_ext.reasoning.tree import RepairMCTS | |
| from unique_ext.reasoning.verify import PytestVerifier | |
| class Garbage: | |
| name, generations = "garbage", 0 | |
| def complete(self, messages, n=1, **kw): | |
| self.generations += n | |
| return ["pas de code"] * n | |
| g, v = Garbage(), PytestVerifier(cache=None) | |
| out = RepairMCTS().solve(request(), Plan("complexe", "mcts", budget=4), g, v, build_for("x", TESTS)) | |
| # le texte brut devient solution.py (règle de parse_files) : la porte JEV le refuse sans lancer pytest | |
| assert len(out) == 1 and not out[0].ok and v.runs == 0 and v.compile_rejects == 1 | |
| assert g.generations == 4 | |
| def test_router_is_hybrid_and_escalates_after_a_failed_attempt(): | |
| from unique_ext.reasoning.router import ModeRouter | |
| simple = ModeRouter(max_budget=6).plan("additionner", {"t.py": "def test_a():\n assert 1\n"}, 1) | |
| assert (simple.level, simple.mode, simple.budget) == ("simple", "direct", 1) | |
| retry = ModeRouter(max_budget=6).plan("additionner", {"t.py": "def test_a():\n assert 1\n"}, 2) | |
| assert retry.level == "moyenne" and retry.budget == 3 | |
| hard = ModeRouter(max_budget=4).plan("arbre récursif, graphe, dijkstra et programmation dynamique " * 20, | |
| {"t.py": "def test_a():\n assert 1\n" * 12}, 1) | |
| assert hard.mode == "mcts" and hard.budget == 4 | |
| # ================================================================= ancrage | |
| def test_api_check_flags_invented_names_only(): | |
| from unique_ext.grounding.api_check import ApiCheck | |
| code = "from math import sqrtt, sqrt\nimport json\nfrom concurrent import futures\nimport helpers\n" | |
| assert ApiCheck().check({"solution.py": code, "helpers.py": "X = 1\n"}) == ["API introuvable : math.sqrtt"] | |
| def test_citations_must_come_from_web_notes(): | |
| from unique_ext.grounding.citations import CitationCheck | |
| code = "# source : https://docs.python.org/3/library/heapq.html.\n# voir https://exemple.invente/page\nURL = 'https://x.y'\n" | |
| notes = ["heapq : file de priorité (https://docs.python.org/3/library/heapq.html)"] | |
| issues = CitationCheck().check({"solution.py": code}, notes) | |
| assert issues == ["citation non sourcée dans solution.py : https://exemple.invente/page"] | |
| # ================================================================= mémoire filtrée | |
| def test_gated_memory_shows_only_close_but_not_identical_examples(): | |
| from mouh.learning.memory import TaskMemory | |
| from unique_ext.memory.gated import GatedMemory | |
| items = [{"task_hash": "a", "instruction": "trier une liste d'entiers par ordre croissant", "files": {}}, | |
| {"task_hash": "b", "instruction": "trier une liste d'entiers par ordre croissant sans doublons", | |
| "files": {}}, | |
| {"task_hash": "d", "instruction": "trier une liste de nombres par ordre croissant", "files": {}}, | |
| {"task_hash": "e", "instruction": "trier des fichiers", "files": {}}, | |
| {"task_hash": "c", "instruction": "calculer la météo", "files": {}}] | |
| loads = [] | |
| mem = GatedMemory(lambda: loads.append(1) or TaskMemory(items), low=0.3, high=0.8, k=3) | |
| q = "trier une liste de nombres par ordre croissant" | |
| assert mem.examples(q, {}, "simple") == [] # tâche simple : aucun exemple | |
| got = [g["task_hash"] for g in mem.examples(q, {}, "moyenne")] | |
| # d : quasi-doublon (souffle la réponse) ; e et c : trop éloignés pour aider | |
| assert sorted(got) == ["a", "b"] and loads == [1] | |
| # ================================================================= session côté Space | |
| def test_extension_session_runs_inside_the_mouh_loop(monkeypatch): | |
| import mouh.service as S | |
| from unique_ext.reasoning.session import ExtSession | |
| label = "UNIQUE · test" | |
| session = ExtSession(None, ScriptedBackend(), grounders=()) | |
| monkeypatch.setattr(S, "EXT_CHOICES", (label,)) | |
| monkeypatch.setattr(S, "_ext_build", lambda choice, load_unique, web_search=False: (session, "scripté")) | |
| snaps = list(S.stream_session("additionner a et b", "", TESTS["test_solution.py"], max_attempts=2, | |
| generator_choice=label, record=False)) | |
| final = snaps[-1] | |
| assert "return a + b" in final["final_code"] | |
| assert session.info["essais"] and session.info["essais"][-1]["tests_ok"] | |
| assert json.loads(final["report"])["reflexion"]["extension"] == "adaptatif" | |
| def test_extension_session_falls_back_when_a_brick_fails(): | |
| from unique_ext.reasoning.session import ExtSession | |
| class Boom: | |
| name = "boom" | |
| def solve(self, *a, **k): | |
| raise RuntimeError("panne simulée") | |
| class V38: | |
| def generate(self, request): | |
| return {"solution.py": "def add(a, b):\n return a + b\n"} | |
| s = ExtSession(V38(), ScriptedBackend(), strategy=Boom(), grounders=()) | |
| assert s.generate(request()) == {"solution.py": "def add(a, b):\n return a + b\n"} | |
| assert "panne simulée" in s.info["repli"] | |
| assert ExtSession(None, ScriptedBackend(), strategy=Boom(), grounders=()).generate(request()) is None | |
| # ================================================================= backends | |
| def tiny_qwen(tmp_path_factory): | |
| pytest.importorskip("peft") | |
| from tokenizers import Tokenizer, decoders, models, pre_tokenizers, trainers | |
| from transformers import PreTrainedTokenizerFast, Qwen2Config, Qwen2ForCausalLM | |
| from mouh.learning import prompts as P | |
| d = tmp_path_factory.mktemp("tiny_qwen_ext") | |
| tk = Tokenizer(models.BPE()) | |
| tk.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) | |
| tk.decoder = decoders.ByteLevel() | |
| corpus = [P.SYSTEM_PROMPT, render_files({"solution.py": "def f():\n return 1\n"}), "## Tâche ## Tests assert"] * 20 | |
| tk.train_from_iterator(corpus, trainers.BpeTrainer(vocab_size=400, special_tokens=["<|endoftext|>", "<|im_start|>", | |
| "<|im_end|>"], | |
| initial_alphabet=pre_tokenizers.ByteLevel.alphabet())) | |
| tok = PreTrainedTokenizerFast(tokenizer_object=tk, eos_token="<|im_end|>", pad_token="<|endoftext|>") | |
| tok.chat_template = ("{% for m in messages %}<|im_start|>{{ m['role'] }}\n{{ m['content'] }}<|im_end|>\n{% endfor %}" | |
| "{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}") | |
| tok.save_pretrained(d) | |
| cfg = Qwen2Config(vocab_size=len(tok), hidden_size=32, intermediate_size=64, num_hidden_layers=2, | |
| num_attention_heads=4, num_key_value_heads=2, max_position_embeddings=4096, | |
| eos_token_id=tok.eos_token_id, pad_token_id=tok.pad_token_id, tie_word_embeddings=True) | |
| Qwen2ForCausalLM(cfg).save_pretrained(d) | |
| return str(d) | |
| def test_prompt_lookup_and_draft_keep_the_greedy_output(tiny_qwen): | |
| import torch | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| from unique_ext.backends.hf import HFBackend | |
| torch.manual_seed(0) | |
| gen = SimpleNamespace(model=AutoModelForCausalLM.from_pretrained(tiny_qwen).eval(), | |
| tokenizer=AutoTokenizer.from_pretrained(tiny_qwen), max_new_tokens=24) | |
| msgs = chat_messages("additionner", TESTS, {"solution.py": "def add(a, b):\n return a - b\n"}, ["assert -1 == 5"]) | |
| plain = HFBackend(gen, lookup=0, draft="").complete(msgs, 1) | |
| fast = HFBackend(gen, lookup=10, draft="").complete(msgs, 1) | |
| draft = HFBackend(gen, lookup=0, draft=tiny_qwen).complete(msgs, 1) # le modèle comme son propre brouillon | |
| assert plain == fast == draft | |
| three = HFBackend(gen, lookup=10, draft="").complete(msgs, 3) | |
| assert len(three) == 3 and three[0] == plain[0] | |
| # ================================================================= jobs : export, banc d'essai, pilote | |
| def test_liger_is_skipped_cleanly_without_gpu(monkeypatch): | |
| import torch | |
| from unique_ext import registry | |
| monkeypatch.setenv("UNIQUE_EXT_JOB_LIGER", "1") | |
| monkeypatch.setattr(torch.cuda, "is_available", lambda: False) | |
| logs = [] | |
| assert registry.job_setup(logs.append) == [] and "pas de GPU" in logs[0] | |
| # ================================================================= phases DPO et KTO dans le vrai job | |
| def session(i): | |
| tests = {"test_solution.py": f"from solution import f\n\ndef test_f():\n assert f() == {i}\n"} | |
| th = task_hash(f"tâche {i}", tests) | |
| final = {"solution.py": f"def f():\n return {i}\n"} | |
| return {"schema": "mouh-session/1", "session_id": f"s{i}", "task_hash": th, "split": split_for(th), | |
| "instruction": f"tâche {i}", "tests": tests, "proofs": [], "generator": "x", "status": "certified", | |
| "certified": True, "final_files": final, | |
| "attempts": [{"attempt": 1, "origin": "model", "files": final, "score": 1.0, "certified": True, | |
| "diagnostics": []}]} | |
| def test_dpo_and_kto_phases_run_through_the_same_gate(tiny_qwen, tmp_path, monkeypatch): | |
| pytest.importorskip("trl") | |
| job = load("jobs/train_unique.py", "train_unique_ext_t2") | |
| data, out = tmp_path / "data", tmp_path / "out" | |
| data.mkdir() | |
| sessions = [session(i) for i in range(60)] | |
| (data / "s.jsonl").write_text("\n".join(json.dumps(s) for s in sessions)) | |
| fails = [{"task_hash": s["task_hash"], "instruction": s["instruction"], "tests": s["tests"], | |
| "files": {"solution.py": "def f():\n return -1\n"}, "pytest_output": "E assert -1 == 1"} | |
| for s in sessions if s["split"] == "train"][:24] | |
| (out / "failures").mkdir(parents=True) | |
| (out / "failures" / "f.jsonl").write_text("\n".join(json.dumps(f) for f in fails)) | |
| monkeypatch.setenv("UNIQUE_PHASES", "dpo:20,kto:20") | |
| monkeypatch.setenv("UNIQUE_EXT_PATH", str(ROOT)) | |
| monkeypatch.setenv("UNIQUE_EXT_PREF_BATCH", "2") | |
| monkeypatch.setenv("UNIQUE_EXT_PREF_ACCUM", "1") | |
| fast = ["--min-heldout", "1", "--sft-chunk", "4", "--batch-size", "2", "--grad-accum", "1", "--max-len", "512", | |
| "--grpo-steps", "0", "--max-new-tokens", "8", "--progress-every", "0", "--benchmark", "0", | |
| "--memory-ab", "0", "--promotion-margin", "1", "--measure-best-of", "0", "--save-failures", "0"] | |
| assert job.main(["--local-data", str(data), "--local-out", str(out), "--base-model", tiny_qwen, "--cycles", "1", | |
| *fast]) == 0 | |
| st = json.loads((out / "state.json").read_text()) | |
| hist = {h["phase"]: h for h in st["history"]} | |
| assert list(hist) == ["sft", "dpo", "kto"], [h.get("phase") for h in st["history"]] | |
| assert hist["dpo"]["train"]["pairs"] == 20 and hist["dpo"]["train"]["steps"] > 0 | |
| assert hist["kto"]["train"]["examples"] >= 16 and hist["kto"]["train"]["steps"] > 0 | |
| assert all("candidate" in h for h in hist.values()) # chaque phase est examinée par la porte | |
| def test_dpo_uses_the_promoted_adapter_as_reference_and_saves_only_the_policy(tiny_qwen, tmp_path): | |
| pytest.importorskip("trl") | |
| from peft import LoraConfig, PeftModel, get_peft_model | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| from unique_ext.contracts import PhaseContext | |
| from unique_ext.training.dpo import DPOPhase | |
| tok = AutoTokenizer.from_pretrained(tiny_qwen) | |
| promoted = tmp_path / "promoted" | |
| get_peft_model(AutoModelForCausalLM.from_pretrained(tiny_qwen), | |
| LoraConfig(r=4, lora_alpha=8, task_type="CAUSAL_LM", target_modules=["q_proj", "v_proj"]) | |
| ).save_pretrained(promoted) | |
| policy = PeftModel.from_pretrained(AutoModelForCausalLM.from_pretrained(tiny_qwen), str(promoted), is_trainable=True) | |
| sessions = [session(i) for i in range(40)] | |
| train = [s for s in sessions if s["split"] == "train"] | |
| fails = [{"task_hash": s["task_hash"], "instruction": s["instruction"], "tests": s["tests"], | |
| "files": {"solution.py": "def f():\n return -1\n"}, "pytest_output": "E"} for s in train] | |
| ctx = PhaseContext(args=SimpleNamespace(max_len=512, seed=0), model=policy, tok=tok, sessions=sessions, | |
| frozen=set(), out_dir=tmp_path / "dpo", cycle=1, run_id="t", token=None, progress=None, | |
| chat_messages=chat_messages, render_files=render_files, run_tests=None, | |
| load_failures=lambda n: fails[:n], make_pytest_reward=None, adapter=str(promoted)) | |
| info = DPOPhase().run(ctx, 16) | |
| assert info["pairs"] == 16 and info["steps"] > 0 and info["reference"] == "copie de l'adaptateur promu" | |
| assert list(policy.peft_config) == ["default"] # « ref » retiré : seul l'adaptateur entraîné est enregistré | |
| policy.save_pretrained(tmp_path / "saved") | |
| assert not (tmp_path / "saved" / "ref").exists() | |
| def test_app_menu_lists_enabled_extensions(monkeypatch): | |
| import app | |
| import mouh.service as S | |
| monkeypatch.setenv("UNIQUE_EXT_TREE", "1") | |
| monkeypatch.setenv("UNIQUE_EXT_GGUF", "1") # v4.1 : retiré, sans effet | |
| try: | |
| importlib.reload(S) | |
| importlib.reload(app) # l'interface se construit, le générateur étendu ajouté en fin de menu | |
| assert app.GENERATOR_CHOICES[:4] == V38_CHOICES and len(app.GENERATOR_CHOICES) == 5 | |
| finally: | |
| monkeypatch.delenv("UNIQUE_EXT_TREE") | |
| monkeypatch.delenv("UNIQUE_EXT_GGUF") | |
| importlib.reload(S) | |
| importlib.reload(app) | |
| # ================================================================= mise à jour par un seul zip (téléphone) | |
| class FakeHub: | |
| """Dépôt du Space versionné en mémoire + dataset d'état : ce que l'updater lit et écrit.""" | |
| def __init__(self, tmp: Path, files: dict[str, bytes]): | |
| self.tmp, self.revs, self.head, self.commits, self.dataset = tmp, {"r0": dict(files)}, "r0", [], {} | |
| def repo_info(self, repo, repo_type=None): | |
| return SimpleNamespace(sha=self.head) | |
| def hf_hub_download(self, repo, path, repo_type=None, revision=None): | |
| store = self.dataset if repo_type == "dataset" else self.revs[revision or self.head] | |
| if path not in store: | |
| raise FileNotFoundError(path) | |
| out = self.tmp / f"dl_{len(list(self.tmp.iterdir()))}" | |
| out.write_bytes(store[path]) | |
| return str(out) | |
| def create_commit(self, repo_id, repo_type, operations, commit_message, parent_commit=None): | |
| assert parent_commit in (None, self.head) | |
| files = dict(self.revs[self.head]) | |
| for op in operations: | |
| if hasattr(op, "path_or_fileobj"): | |
| files[op.path_in_repo] = op.path_or_fileobj | |
| else: | |
| files.pop(op.path_in_repo, None) | |
| self.head = f"r{len(self.revs)}" | |
| self.revs[self.head] = files | |
| self.commits.append((commit_message, [op.path_in_repo for op in operations])) | |
| return SimpleNamespace(oid=self.head) | |
| def upload_file(self, path_or_fileobj, path_in_repo, repo_id, repo_type, commit_message): | |
| self.dataset[path_in_repo] = path_or_fileobj | |
| def make_zip(entries: dict[str, str], prefix: str = "") -> bytes: | |
| import io | |
| import zipfile | |
| buf = io.BytesIO() | |
| with zipfile.ZipFile(buf, "w") as z: | |
| for name, text in entries.items(): | |
| z.writestr(prefix + name, text) | |
| return buf.getvalue() | |
| def manifest_for(files: dict[str, str], version: str) -> str: | |
| from unique_ext.updater import normalized_hash | |
| return json.dumps({"version": version, "files": {p: normalized_hash(t) for p, t in files.items()}}) | |
| def space(tmp_path): | |
| root = tmp_path / "app" | |
| current = {"app.py": "print('v1')\n", "unique_ext/a.py": "A = 1\n", "requirements.txt": "gradio\n"} | |
| for p, t in current.items(): | |
| (root / p).parent.mkdir(parents=True, exist_ok=True) | |
| (root / p).write_text(t) | |
| (tmp_path / "dl").mkdir() | |
| hub = FakeHub(tmp_path / "dl", {p: t.encode() for p, t in current.items()}) | |
| return root, hub | |
| def test_update_installs_in_one_commit_is_idempotent_and_rolls_back(space, monkeypatch): | |
| from unique_ext import updater as U | |
| root, hub = space | |
| new = {"app.py": "print('v2')\n", "unique_ext/a.py": "A = 1\n", "unique_ext/b.py": "B = 2\n"} | |
| tracked = {**new, "requirements.txt": "gradio\n"} | |
| zbytes = make_zip({**new, "mouh/learning/manifest.json": manifest_for(tracked, "v9.9")}, prefix="unique-v9.9/") | |
| hub.revs["r0"]["updates/u.zip"] = zbytes | |
| state = {} | |
| rs, ws = (lambda: state.get("s")), (lambda obj: state.update(s=obj)) | |
| msg = U.install(hub, "RDS777/TARGET_SPACE", "updates/u.zip", root=root, read_state=rs, write_state=ws) | |
| assert "v9.9 installée" in msg and len(hub.commits) == 1 | |
| assert sorted(hub.commits[0][1]) == ["app.py", "mouh/learning/manifest.json", "unique_ext/b.py"] # a.py inchangé | |
| assert hub.revs[hub.head]["app.py"] == b"print('v2')\n" and state["s"]["new_files"] == [ | |
| "mouh/learning/manifest.json", "unique_ext/b.py"] | |
| for p, t in new.items(): # le Space s'est reconstruit avec les nouveaux fichiers | |
| (root / p).write_text(t) | |
| (root / "mouh/learning").mkdir(parents=True) | |
| (root / "mouh/learning/manifest.json").write_text(manifest_for(tracked, "v9.9")) | |
| again = U.install(hub, "RDS777/TARGET_SPACE", "updates/u.zip", root=root, read_state=rs, write_state=ws) | |
| assert "déjà en place" in again and len(hub.commits) == 1 # pas de boucle de redémarrages | |
| back = U.rollback(hub, "RDS777/TARGET_SPACE", read_state=rs, write_state=ws) | |
| files = hub.revs[hub.head] | |
| assert "retour arrière fait" in back and files["app.py"] == b"print('v1')\n" | |
| assert "unique_ext/b.py" not in files and "mouh/learning/manifest.json" not in files | |
| assert "déjà fait" in U.rollback(hub, "RDS777/TARGET_SPACE", read_state=rs, write_state=ws) | |
| def test_update_removes_retired_files_and_rollback_restores_them(space): | |
| from unique_ext import updater as U | |
| root, hub = space | |
| (root / "unique_ext/old.py").write_text("OLD = 1\n") | |
| hub.revs["r0"]["unique_ext/old.py"] = b"OLD = 1\n" | |
| new = {"app.py": "print('v2')\n", "unique_ext/a.py": "A = 1\n"} | |
| man = json.loads(manifest_for({**new, "requirements.txt": "gradio\n"}, "v4.1")) | |
| man["removed"] = ["unique_ext/old.py", "unique_ext/absent.py"] | |
| hub.revs["r0"]["updates/u.zip"] = make_zip({**new, "mouh/learning/manifest.json": json.dumps(man)}) | |
| state = {} | |
| rs, ws = (lambda: state.get("s")), (lambda obj: state.update(s=obj)) | |
| msg = U.install(hub, "RDS777/TARGET_SPACE", "updates/u.zip", root=root, read_state=rs, write_state=ws) | |
| assert "1 retiré(s)" in msg and "unique_ext/old.py" not in hub.revs[hub.head] | |
| assert state["s"]["removed"] == ["unique_ext/old.py"] and len(hub.commits) == 1 | |
| (root / "unique_ext/old.py").unlink() # le Space s'est reconstruit sans le fichier retiré | |
| for p, t in new.items(): | |
| (root / p).write_text(t) | |
| (root / "mouh/learning").mkdir(parents=True, exist_ok=True) | |
| (root / "mouh/learning/manifest.json").write_text(json.dumps(man)) | |
| again = U.install(hub, "RDS777/TARGET_SPACE", "updates/u.zip", root=root, read_state=rs, write_state=ws) | |
| assert "déjà en place" in again and len(hub.commits) == 1 | |
| assert "retour arrière fait" in U.rollback(hub, "RDS777/TARGET_SPACE", read_state=rs, write_state=ws) | |
| assert hub.revs[hub.head]["unique_ext/old.py"] == b"OLD = 1\n" | |
| bad = make_zip({"mouh/learning/manifest.json": json.dumps( | |
| {"version": "x", "files": {}, "removed": ["../etc/passwd", "Dockerfile", "unique_ext/../app.py"]})}) | |
| b = U.read_bundle(bad, root) | |
| assert b.removed == [] and sum("suppression refusée" in p for p in b.problems) == 3 | |
| def test_update_refuses_unsafe_or_incomplete_bundles(space, entries, reason): | |
| from unique_ext import updater as U | |
| root, hub = space | |
| hub.revs["r0"]["updates/bad.zip"] = make_zip(entries) | |
| state = {} | |
| msg = U.install(hub, "RDS777/TARGET_SPACE", "updates/bad.zip", root=root, read_state=lambda: None, | |
| write_state=lambda o: state.update(s=o)) | |
| assert "refusée" in msg and reason in msg and not hub.commits and state["s"]["ok"] is False | |
| def test_update_does_nothing_unless_asked(monkeypatch): | |
| from unique_ext.updater import ensure_update | |
| monkeypatch.delenv("UNIQUE_EXT_INSTALL", raising=False) | |
| monkeypatch.delenv("UNIQUE_EXT_ROLLBACK", raising=False) | |
| assert ensure_update(None, {"token": "tok"})[0] == "mise à jour : aucune demandée" | |
| # ================================================================= livraison | |
| def test_release_manifest_is_current_and_release_zip_passes_the_updater(tmp_path): | |
| import io | |
| import zipfile | |
| br = load("scripts/build_release.py", "build_release_t") | |
| assert br.check() == [] # le manifest livré correspond aux fichiers (sinon : python scripts/build_release.py) | |
| from unique_ext.updater import read_bundle | |
| man = br.build_manifest("vtest") | |
| buf = io.BytesIO() | |
| with zipfile.ZipFile(buf, "w") as z: | |
| for rel in br.runtime_files(): | |
| z.write(ROOT / rel, rel) | |
| z.writestr(br.MANIFEST, json.dumps(man)) | |
| b = read_bundle(buf.getvalue(), root=tmp_path) # Space vide : tout doit venir du zip | |
| assert b.problems == [] and b.version == "vtest" and len(b.files) == len(man["files"]) + 1 | |
| # ================================================================= v3.10 : passage au 7B | |
| def test_space_refuses_to_load_a_7b_in_float32_on_cpu(monkeypatch): | |
| import torch | |
| from mouh.learning.unique_generator import UniqueGenerator, model_size_b | |
| assert model_size_b("Qwen/Qwen2.5-Coder-7B-Instruct") == 7.0 and model_size_b("/tmp/tiny") is None | |
| monkeypatch.setenv("UNIQUE_BASE_MODEL", "Qwen/Qwen2.5-Coder-7B-Instruct") | |
| monkeypatch.setattr(torch.cuda, "is_available", lambda: False) | |
| with pytest.raises(RuntimeError, match="GPU"): | |
| UniqueGenerator.from_env() | |
| def test_training_job_loads_large_models_in_bf16_only_on_a_suitable_gpu(monkeypatch): | |
| import torch | |
| job = load("jobs/train_unique.py", "train_unique_ext_t3") | |
| big, small = SimpleNamespace(base_model="Qwen/Qwen2.5-Coder-7B-Instruct"), SimpleNamespace(base_model="/tmp/tiny") | |
| monkeypatch.setattr(torch.cuda, "is_available", lambda: True) | |
| monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: (8, 9)) | |
| monkeypatch.setattr(torch.cuda, "get_device_properties", lambda *a: SimpleNamespace(total_memory=24 * 2**30)) | |
| assert job._load_dtype(big) == torch.bfloat16 and job._load_dtype(small) == torch.float32 | |
| assert job.gpu_problem(7.0) is None # L4 : 24 Go, bf16 | |
| monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: (7, 5)) | |
| assert "UNIQUE_FLAVOR=l4x1" in job.gpu_problem(7.0) # T4 : refusé avant tout chargement | |
| assert job._load_dtype(big) == torch.float32 | |
| # ================================================================= v3.10.1 : A10G small, coût réel, état des jobs | |
| def test_large_models_load_straight_onto_the_gpu(monkeypatch): | |
| import torch | |
| import transformers | |
| job = load("jobs/train_unique.py", "train_unique_ext_t4") | |
| seen = {} | |
| monkeypatch.setattr(torch.cuda, "is_available", lambda: True) | |
| monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: (8, 6)) | |
| monkeypatch.setattr(transformers.AutoModelForCausalLM, "from_pretrained", | |
| lambda name, **kw: seen.update(kw) or "modèle") | |
| assert job._load_base(SimpleNamespace(base_model="Qwen/Qwen2.5-Coder-7B-Instruct"), None) == "modèle" | |
| assert seen["device_map"] == {"": 0} and seen["dtype"] == torch.bfloat16 # jamais 15 Go en RAM d'un coup | |
| def test_progress_tracks_the_real_cost_of_each_run(tmp_path, monkeypatch): | |
| job = load("jobs/train_unique.py", "train_unique_ext_t5") | |
| store = job.Store(local_out=str(tmp_path), adapter_repo="x", bucket="b", token=None) | |
| today = job._now().strftime("%Y-%m-%d") | |
| store.write_progress({"day": today, "gpu_hours_today": 16.0, "adapter": "x"}, True) # 16 h d'avant, sur T4 | |
| monkeypatch.setenv("UNIQUE_GPU_PRICE_PER_H", "1.00") | |
| pr = job.Progress(store, run_id="r", max_hours=1, daily_budget_h=24, min_interval_s=0, adapter="x") | |
| pr.t0 -= 1800 # une demi-heure sur A10G small | |
| pr.update(important=True) | |
| st = store.read_progress() | |
| assert st["gpu_cost_before_run"] == 6.4 and abs(st["gpu_cost_today"] - 6.9) < 0.01 and st["gpu_price_per_h"] == 1.0 | |
| # ================================================================= v3.10.2 : professeur, faux refus de jeton | |
| # ================================================================= v3.10.3 : jeton renouvelé, doublons, droits Jobs | |
| # ================================================================= v3.10.4 : mesures annexes après l'examen | |
| def test_measures_free_the_gpu_before_loading_another_7b(monkeypatch): | |
| job = load("jobs/train_unique.py", "train_unique_ext_measures") | |
| events = [] | |
| monkeypatch.setattr(job, "_free_gpu", lambda: events.append("libère")) | |
| monkeypatch.setattr(job, "_load_base", lambda a, token: events.append("charge base") or "base") | |
| monkeypatch.setattr(job, "load_policy", lambda a, tok, token, src: events.append("charge meilleur") or "meilleur") | |
| monkeypatch.setattr(job, "evaluate", lambda model, *r, **k: {"passed": 1, "n": 2, "rate": 0.5, "ci95": [0, 1]}) | |
| store = SimpleNamespace(adapter_source=lambda: "me/a", write_json=lambda *r, **k: None) | |
| progress = SimpleNamespace(update=lambda **k: None, phase=lambda *r, **k: None) | |
| a = SimpleNamespace(measure_best_of=1, memory_ab=0) | |
| state = {"best": {"passed": 85, "n": 100, "source": "sft r c1"}, "base": {"passed": 80, "n": 100}, "heldout_sig": "s"} | |
| job.measure_best(a, None, None, store, state, [], [], [{"task_hash": "humaneval:0"}], progress, "r") | |
| # vu le 05/11 : le 7B de GRPO occupait encore le GPU quand le 7B de base a été chargé pour HumanEval | |
| assert events == ["libère", "charge base", "libère", "charge meilleur", "libère"] | |
| assert state["measures"]["humaneval"]["passed"] == 1 | |
| def test_a_failing_measure_never_stops_the_run(tiny_qwen, tmp_path, monkeypatch): | |
| pytest.importorskip("trl") | |
| from test_learning import FAST, write_sessions | |
| job = load("jobs/train_unique.py", "train_unique_ext_measures_fail") | |
| calls = [] | |
| def oom(*a, **k): | |
| calls.append(1) | |
| raise RuntimeError("CUDA out of memory. Tried to allocate 1.02 GiB") | |
| monkeypatch.setattr(job, "measure_best", oom) | |
| data, out = tmp_path / "data", tmp_path / "out" | |
| write_sessions(data, 40) | |
| args = ["--local-data", str(data), "--local-out", str(out), "--base-model", tiny_qwen, "--cycles", "2", *FAST] | |
| args[args.index("--grpo-steps") + 1] = "0" | |
| assert job.main(args) == 0 and len(calls) == 2 # le passage continue : 2 cycles faits | |
| st = json.loads((out / "state.json").read_text()) | |
| assert st["last_error"]["phase"] == "mesures" and "out of memory" in st["measures"]["error"] | |
| assert [h["phase"] for h in st["history"]] == ["sft", "sft"] | |
| assert json.loads((out / "progress.json").read_text())["phase"] == "fin" | |
| # ================================================================= v3.10.5 : maître fiable, 409 lisible | |
| # ================================================================= v3.10.6 : révision du maître, énoncé vide | |