"""v4.0 : mode local (un seul Space, aucune inférence en ligne). Aucun réseau, aucun GPU : faux processus, faux Hub.""" from __future__ import annotations import importlib import importlib.util import json import os import threading import time from pathlib import Path from types import SimpleNamespace import pytest from mouh.learning import dashboard from mouh.learning.local_ingest import ingest_jsonl from mouh.learning.local_store import LocalStore, hf_home_for, resolve_root from mouh.learning.local_trainer import LocalTrainer, gpu_status, import_from_hub, suspend_remote_jobs from mouh.learning.prompts import chat_messages, render_files from mouh.learning.records import split_for, task_hash ROOT = Path(__file__).resolve().parents[1] def load(rel: str, name: str): spec = importlib.util.spec_from_file_location(name, ROOT / rel) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod JOB = load("jobs/train_unique.py", "train_unique_local") TESTS = ("from solution import maximum\n\n\ndef test_maximum():\n assert maximum([3, 1, 2]) == 3\n\n\n" "def test_negatifs():\n assert maximum([-5, -2, -9]) == -2\n") BUGGY = "def maximum(xs):\n m = 0\n for x in xs:\n if x > m:\n m = x\n return m\n" GOOD = "def maximum(xs):\n m = xs[0]\n for x in xs[1:]:\n if x > m:\n m = x\n return m\n" def instruction_for(split: str, stem: str = "Écrire maximum(xs) pour une liste non vide de nombres") -> str: for i in range(500): text = f"{stem} (variante {i})." if split_for(task_hash(text, {"test_solution.py": TESTS})) == split: return text raise AssertionError("aucune variante") def line(**kw) -> str: return json.dumps(kw, ensure_ascii=False) # ================================================================= stockage local def test_root_prefers_explicit_then_persistent_storage_then_home(tmp_path): assert resolve_root({"UNIQUE_LOCAL_ROOT": str(tmp_path / "x")}) == tmp_path / "x" persistent = tmp_path / "data" persistent.mkdir() assert resolve_root({}, persistent=persistent) == persistent / "unique" assert resolve_root({"MOUH_DATA_DIR": str(tmp_path / "h")}, persistent=tmp_path / "absent") == tmp_path / "h" / "unique" def test_store_layout_inventory_and_adapter(tmp_path): store = LocalStore(tmp_path / "u", persistent_root=tmp_path / "data") assert store.data_dir.is_dir() and store.failures_dir.is_dir() and not store.persistent (store.data_dir / "a.jsonl").write_text('{"x": 1}\n\n{"x": 2}\n') (store.failures_dir / "f.jsonl").write_text('{"y": 1}\n') assert store.inventory() == {"data": {"fichiers": 1, "lignes": 2}, "failures": {"fichiers": 1, "lignes": 1}} assert not store.has_adapter() and store.adapter_stamp() is None store.adapter_dir.mkdir() (store.adapter_dir / "adapter_config.json").write_text("{}") assert store.has_adapter() and store.adapter_stamp() is not None assert LocalStore(tmp_path / "data" / "unique", persistent_root=tmp_path / "data").persistent # ================================================================= dépôt de JSONL vérifiés def test_a_simple_task_with_a_real_bug_feeds_sft_grpo_and_dpo(tmp_path): from unique_ext.training.pairs import preference_pairs store = LocalStore(tmp_path / "u") instr = instruction_for("train") rep = ingest_jsonl(line(instruction=instr, tests=TESTS, solution=GOOD, level="extrême", rejected=BUGGY) + "\n", "lot1.jsonl", store.data_dir, store.failures_dir) assert (rep.sessions, rep.failures, sum(rep.rejected.values())) == (1, 1, 0) and len(rep.written) == 2 rec = json.loads(next(store.data_dir.glob("upload-*.jsonl")).read_text()) assert rec["certified"] and rec["generator"] == "upload:lot1" and rec["attempts"][0]["level"] == "extrême" assert [a["certified"] for a in rec["attempts"]] == [False, True] and "-2" in rec["attempts"][0]["diagnostics"][0] train, _heldout, stats = JOB.build_examples([rec]) # SFT : génération + réparation, en tête de file assert stats["gen"] == 1 and stats["fix"] == 1 and train[0]["tier"] == 0 assert JOB._train_tasks([rec], [])[0]["level"] == "extrême" # curriculum GRPO job_store = JOB.Store(local_out=str(store.root), adapter_repo="local", bucket="b", token=None) fails = JOB.load_failures(job_store, "local", None, 10) # le job lit l'échec tel quel assert len(fails) == 1 and fails[0]["task_hash"] == rec["task_hash"] ctx = SimpleNamespace(sessions=[rec], frozen=set(), render_files=render_files, chat_messages=chat_messages, load_failures=lambda n: fails) pairs = preference_pairs(ctx, 10) # DPO : logique native, data/ + failures/ assert len(pairs) == 1 and "m = 0" in pairs[0]["rejected"][0]["content"] def test_nothing_enters_without_passing_the_sandbox(tmp_path): store = LocalStore(tmp_path / "u") instr = instruction_for("train") session_bad = {"schema": "mouh-session/1", "certified": True, "instruction": instr, "tests": {"test_solution.py": TESTS}, "final_files": {"solution.py": BUGGY}, "attempts": []} failure_ok = {"task_hash": "t", "instruction": instr, "tests": {"test_solution.py": TESTS}, "files": {"solution.py": GOOD}, "pytest_output": "E"} rows = [ line(instruction=instr, tests=TESTS, solution=BUGGY), # solution fausse line(instruction=instr, tests=TESTS.replace("from solution import maximum", "import math"), solution=GOOD), line(prompt="p", chosen="c", rejected="r"), # invérifiable "{pas du json", json.dumps(session_bad), json.dumps(failure_ok), # « échec » qui passe line(instruction=instr, tests=TESTS, solution=GOOD, rejected=GOOD.replace("m = xs[0]", "m = xs[0] # idem")), line(instruction=instr, tests=TESTS, solution=GOOD), # doublon de la précédente ] rep = ingest_jsonl("\n".join(rows), "mixte.jsonl", store.data_dir, store.failures_dir) assert rep.sessions == 1 and rep.failures == 0 assert rep.rejected == {"la solution échoue à ses tests": 2, "les tests n'importent pas solution": 1, "paire prompt/chosen/rejected sans tests (invérifiable)": 1, "JSON illisible": 1, "l'échec annoncé passe les tests": 1, "doublon dans le fichier": 1} assert "ligne 1" in rep.markdown() and "refusées" in rep.markdown() # sans réexécution, une session déjà certifiée par sa source est reprise telle quelle rep2 = ingest_jsonl(json.dumps(session_bad), "confiance.jsonl", store.data_dir, store.failures_dir, verify=False) assert rep2.sessions == 1 def test_heldout_tasks_are_kept_but_reported(tmp_path): store = LocalStore(tmp_path / "u") rep = ingest_jsonl(line(instruction=instruction_for("heldout"), tests=TESTS, solution=GOOD), "h.jsonl", store.data_dir, store.failures_dir) assert rep.sessions == 1 and rep.heldout == 1 and "examen" in rep.markdown() # ================================================================= entraînement local class FakeProc: def __init__(self, cmd, **kw): self.cmd, self.kw, self.code = cmd, kw, None self.released = threading.Event() self.terminated = False def poll(self): return self.code def wait(self, timeout=None): if not self.released.wait(timeout if timeout is not None else 10): raise AssertionError("processus jamais terminé") return self.code def finish(self, code=0): self.code = code self.released.set() def terminate(self): self.terminated = True self.finish(-15) def kill(self): self.finish(-9) def wait_for(cond, timeout=5.0): end = time.time() + timeout while time.time() < end: if cond(): return True time.sleep(0.01) return False def test_trainer_command_runs_the_job_on_local_data_without_any_hub_call(tmp_path): store = LocalStore(tmp_path / "u") t = LocalTrainer(store, env={"UNIQUE_PHASES": "kto:100"}) cmd = t.command() for flag, value in (("--local-out", str(store.root)), ("--local-data", str(store.data_dir)), ("--space-id", ""), ("--daily-gpu-hours", "24"), ("--benchmark", "0")): assert cmd[cmd.index(flag) + 1] == value env = t.child_env() assert env["UNIQUE_EXT_PATH"] == str(ROOT) and env["UNIQUE_PHASES"] == "kto:100" t.dpo = True assert t.child_env()["UNIQUE_PHASES"] == "kto:100,dpo:200" assert "--benchmark" not in LocalTrainer(store, env={"UNIQUE_BENCHMARK": "1"}).command() def test_trainer_refuses_without_gpu_and_runs_in_a_loop_with_one(tmp_path): store = LocalStore(tmp_path / "u") procs = [] def popen(cmd, **kw): procs.append(FakeProc(cmd, **kw)) return procs[-1] no_gpu = LocalTrainer(store, gpu=lambda: {"ok": False, "detail": "aucun GPU"}, popen=popen) assert "impossible" in no_gpu.start() and procs == [] t = LocalTrainer(store, gpu=lambda: {"ok": True, "detail": "NVIDIA A10G"}, popen=popen, restart_delay_s=0.01, quick_exit_s=0) assert "A10G" in t.start() and wait_for(lambda: t.running) assert procs[0].kw["env"]["UNIQUE_EXT_PATH"] == str(ROOT) and procs[0].kw["cwd"] == str(ROOT) assert "déjà" in t.start() procs[0].finish(0) # fin normale d'un passage : relance automatique assert wait_for(lambda: len(procs) == 2 and t.running) assert "arrêté" in t.stop(wait_s=1) and procs[1].terminated and not t.running assert wait_for(lambda: not t._thread.is_alive()) and len(procs) == 2 # plus de relance après l'arrêt assert "passage 2" in t.log_tail() def test_gpu_probe_never_touches_cuda_in_the_interface(): def runner(answers): def run(cmd, **kw): out = answers.pop(0) if isinstance(out, Exception): raise out return SimpleNamespace(returncode=out[0], stdout=out[1]) return run assert gpu_status(runner([(0, "NVIDIA A10G, 23028 MiB\n")]))["ok"] assert gpu_status(runner([FileNotFoundError("nvidia-smi"), (0, "NVIDIA A10G\n")]))["detail"] == "NVIDIA A10G" assert not gpu_status(runner([FileNotFoundError("nvidia-smi"), (0, "\n")]))["ok"] def test_old_hub_jobs_are_suspended_and_running_ones_cancelled(): class Api: def __init__(self): self.suspended, self.cancelled = [], [] def list_scheduled_jobs(self, labels): if labels["app"] == "unique-train": return [SimpleNamespace(id="s1", suspend=False)] if labels["app"] == "unique-teacher": return [SimpleNamespace(id="s2", suspend=True)] # déjà suspendu : on n'y touche pas return [] def suspend_scheduled_job(self, scheduled_job_id): self.suspended.append(scheduled_job_id) def list_jobs(self, status, labels): return [SimpleNamespace(id="j1")] if labels["app"] == "unique-train" else [] def cancel_job(self, job_id): self.cancelled.append(job_id) api = Api() assert suspend_remote_jobs(None, api=api) == ["unique-train suspendu"] and api.cancelled == [] api = Api() done = suspend_remote_jobs(None, api=api, cancel_running=True) assert api.suspended == ["s1"] and api.cancelled == ["j1"] and "unique-train en cours annulé" in done assert suspend_remote_jobs(None) == [] # sans jeton : rien n'est tenté def test_one_time_import_brings_back_data_adapter_and_frozen_exam(tmp_path): store = LocalStore(tmp_path / "u") def download(repo_id, local_dir, allow_patterns, token=None, repo_type=None): root = Path(local_dir) if repo_type == "dataset": (root / "data").mkdir(parents=True, exist_ok=True) (root / "data" / "teacher-1.jsonl").write_text("{}\n") (root / "failures").mkdir(parents=True, exist_ok=True) (root / "failures" / "r1.jsonl").write_text("{}\n") else: root.mkdir(parents=True, exist_ok=True) for name in ("adapter_config.json", "adapter_model.safetensors"): (root / name).write_text("x") (root / "state.json").write_text(json.dumps({"heldout_tasks": [1, 2], "best": {"passed": 85}})) done = import_from_hub(store, "me/data", "me/adapter", None, download=download) assert len(done) == 3 and store.has_adapter() and store.read_json("state.json")["best"]["passed"] == 85 assert store.inventory()["failures"]["lignes"] == 1 and not (store.root / "import-adapter").exists() # ================================================================= tableau de bord STATE = {"base": {"passed": 80, "n": 100}, "best": {"passed": 85, "n": 100, "source": "sft r c1"}, "history": [ {"at": "2026-11-04T22:39:49", "cycle": 1, "phase": "sft", "train": {"loss": 0.1822}, "candidate": {"rate": 0.85}, "best_before": {"rate": 0.8}, "base": {"rate": 0.8}, "promoted": True}, {"at": "2026-11-04T23:44:53", "cycle": 1, "phase": "grpo", "train": {"loss": 0.0158}, "candidate": {"rate": 0.85}, "best_before": {"rate": 0.85}, "base": {"rate": 0.8}, "promoted": False}, {"at": "2026-11-05T00:46:23", "cycle": 2, "phase": "sft", "train": {"loss": 0.1501}, "candidate": {"rate": 0.81}, "best_before": {"rate": 0.85}, "base": {"rate": 0.8}, "promoted": False}]} def test_dashboard_history_curves_and_best_losses(tmp_path): rows = dashboard.cycle_rows(STATE) assert [r["candidat"] for r in rows] == [85.0, 85.0, 81.0] and rows[0]["promu"] assert dashboard.best_losses(rows) == {"sft": 0.1501, "grpo": 0.0158} pf, lf = dashboard.pass_frame(rows), dashboard.loss_frame(rows) assert list(pf.columns) == ["n", "taux", "série"] and len(pf) == 9 and len(lf) == 3 assert dashboard.table(rows)[0][0] == "2026-11-05 00:46" and dashboard.table(rows)[-1][-1] == "oui" store = LocalStore(tmp_path / "u", persistent_root=tmp_path / "data") trainer = LocalTrainer(store, gpu=lambda: {"ok": False, "detail": "aucun GPU"}) md = dashboard.status_markdown(store=store, trainer=trainer, gpu={"detail": "aucun GPU"}, state=STATE, progress={"phase": "sft", "step": 3, "total": 125, "status": "SFT", "metrics": {"loss": 0.2}}, inventory=store.inventory()) assert "non persistant" in md and "base 80" in md and "meilleur 85" in md and "historique" in md and "sft 0.1501" in md assert "phase sft 3/125" in md assert dashboard.cycle_rows(None, {"history": [{"at": "x", "candidate": 0.84, "best": 0.85, "base": 0.8}]})[0][ "candidat"] == 84.0 def test_interface_sessions_stay_local(tmp_path): from mouh.learning import recorder rec = recorder.use_folder(tmp_path / "u") try: rec.record({"schema": "mouh-session/1"}) assert not rec.remote and len(list((tmp_path / "u" / "data").glob("sessions-*.jsonl"))) == 1 assert recorder.get_recorder() is rec finally: recorder._RECORDER = None # ================================================================= interface @pytest.fixture def local_app(tmp_path, monkeypatch): monkeypatch.setenv("UNIQUE_LOCAL_ROOT", str(tmp_path / "u")) monkeypatch.setenv("UNIQUE_ADMIN_KEY", "secret") import app importlib.reload(app) yield app monkeypatch.delenv("UNIQUE_LOCAL_ROOT") importlib.reload(app) def test_interface_has_no_online_inference_and_keeps_the_verification_api(local_app): app = local_app assert "Modèle M.O.U.H." not in app.UI_CHOICES and app.UI_CHOICES[0] == "Vérification seule (code fourni)" named = app.demo.get_api_info()["named_endpoints"] assert [p["parameter_name"] for p in named["/run"]["parameters"]] == [ "instruction", "code", "tests", "proofs", "max_attempts", "generator", "record"] src = (ROOT / "app.py").read_text(encoding="utf-8") assert "ddgs" not in src and "vllm" not in src.lower() and "autopilot" not in src assert "ddgs" not in (ROOT / "requirements.txt").read_text() status, pass_df, loss_df, table, console = app.refresh() assert "Mode local" in status and list(pass_df.columns) == ["n", "taux", "série"] def test_interface_commands_need_the_key_and_uploads_are_verified(local_app, tmp_path): app = local_app assert app.start_cmd("mauvaise", False) == app.REFUSED and app.upload_cmd([], True, "") == app.REFUSED f = tmp_path / "lot.jsonl" f.write_text(line(instruction=instruction_for("train"), tests=TESTS, solution=GOOD) + "\n") msg = app.upload_cmd([str(f)], True, "secret") assert "1 session(s) certifiée(s)" in msg and app.STORE.inventory()["data"]["lignes"] == 1 app.TRAINER.proc = FakeProc(["x"]) # entraînement en cours : le GPU n'est pas partagé out = list(app.run("t", GOOD, TESTS, "", 1, app.UI_CHOICES[1])) assert "GPU occupé" in out[0][1] app.TRAINER.proc.finish(0) def test_a_passage_that_ends_at_once_is_not_relaunched_every_30_seconds(tmp_path): store = LocalStore(tmp_path / "u") procs = [] def popen(cmd, stdout=None, **kw): stdout.write(b"[16:08:59] Fetching 4 files: 25%|##5 | 1/4 [00:09<00:28, 9.39s/it]\n" b"[16:09:00] pas assez de sessions certifiees : 0\n") procs.append(FakeProc(cmd, **kw)) return procs[-1] t = LocalTrainer(store, gpu=lambda: {"ok": True, "detail": "NVIDIA A10G"}, popen=popen, restart_delay_s=0.01, idle_delay_s=30) t.start() assert wait_for(lambda: t.running) procs[0].finish(0) assert wait_for(lambda: "passage très court" in t.log_tail()) time.sleep(0.2) assert len(procs) == 1 and t.last_reason == "[16:09:00] pas assez de sessions certifiees : 0" assert "pas assez de sessions" in t.log_tail() t.stop(wait_s=1) assert wait_for(lambda: not t._thread.is_alive()) # -- v4.1.2 : le cache du modèle reste hors du stockage monté, la RAM est mesurée ------------------------------ def test_model_cache_stays_off_the_mounted_storage_by_default(): root = Path("/data/unique") assert hf_home_for(root, {}) is None assert hf_home_for(root, {"UNIQUE_HF_ON_DATA": "0"}) is None assert hf_home_for(root, {"UNIQUE_HF_ON_DATA": "1"}) == Path("/data/unique/hf") assert hf_home_for(Path("/home/user/data/unique"), {"UNIQUE_HF_ON_DATA": "1"}) is None # dossier non monté def test_app_does_not_point_hf_home_at_the_mounted_storage(monkeypatch): source = (ROOT / "app.py").read_text(encoding="utf-8") assert 'startswith("/data/")' not in source and "hf_home_for(ROOT)" in source def test_ram_report_reads_process_and_container_memory(tmp_path): (tmp_path / "proc/self").mkdir(parents=True) (tmp_path / "proc/self/status").write_text("Name:\tpython\nVmRSS:\t 3145728 kB\n") cg = tmp_path / "sys/fs/cgroup" cg.mkdir(parents=True) (cg / "memory.current").write_text(str(14 * 2**30)) (cg / "memory.stat").write_text(f"anon {2 * 2**30}\nfile {10 * 2**30}\n") assert JOB.ram_report(str(tmp_path)) == "processus 3,0 Go · conteneur 14,0 Go (dont cache de fichiers 10,0 Go)" def test_ram_report_never_raises_when_nothing_is_readable(tmp_path): assert JOB.ram_report(str(tmp_path)) == "mesure indisponible" assert isinstance(JOB.ram_report(), str) # sur la vraie machine : ne lève rien def test_dashboard_uses_the_current_frozen_exam(tmp_path): store = LocalStore(tmp_path / "u", persistent_root=tmp_path / "data") trainer = LocalTrainer(store, gpu=lambda: {"ok": False, "detail": "aucun GPU"}) text = dashboard.status_markdown(store=store, trainer=trainer, gpu={}, state=STATE, progress=None, inventory=store.inventory(), exam_state={"gele": True, "taches": 84, "sig": "current", "base": {"passed": 59}, "service": {"passed": 58}}) assert "84 tâches" in text and "59/84 (70.24 %)" in text and "58/84 (69.05 %)" in text assert "100 tâches" not in text