TARGET_SPACE / tests /test_local.py
RDS777's picture
Upload 39 files
5c24fd3 verified
Raw History Blame Contribute Delete
20.5 kB
"""v4.0 : mode local (un seul Space, aucune inférence en ligne). Aucun réseau, aucun GPU : faux processus, faux Hub."""
from __future__ import annotations
import importlib
import importlib.util
import json
import os
import threading
import time
from pathlib import Path
from types import SimpleNamespace
import pytest
from mouh.learning import dashboard
from mouh.learning.local_ingest import ingest_jsonl
from mouh.learning.local_store import LocalStore, hf_home_for, resolve_root
from mouh.learning.local_trainer import LocalTrainer, gpu_status, import_from_hub, suspend_remote_jobs
from mouh.learning.prompts import chat_messages, render_files
from mouh.learning.records import split_for, task_hash
ROOT = Path(__file__).resolve().parents[1]
def load(rel: str, name: str):
spec = importlib.util.spec_from_file_location(name, ROOT / rel)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
JOB = load("jobs/train_unique.py", "train_unique_local")
TESTS = ("from solution import maximum\n\n\ndef test_maximum():\n assert maximum([3, 1, 2]) == 3\n\n\n"
"def test_negatifs():\n assert maximum([-5, -2, -9]) == -2\n")
BUGGY = "def maximum(xs):\n m = 0\n for x in xs:\n if x > m:\n m = x\n return m\n"
GOOD = "def maximum(xs):\n m = xs[0]\n for x in xs[1:]:\n if x > m:\n m = x\n return m\n"
def instruction_for(split: str, stem: str = "Écrire maximum(xs) pour une liste non vide de nombres") -> str:
for i in range(500):
text = f"{stem} (variante {i})."
if split_for(task_hash(text, {"test_solution.py": TESTS})) == split:
return text
raise AssertionError("aucune variante")
def line(**kw) -> str:
return json.dumps(kw, ensure_ascii=False)
# ================================================================= stockage local
def test_root_prefers_explicit_then_persistent_storage_then_home(tmp_path):
assert resolve_root({"UNIQUE_LOCAL_ROOT": str(tmp_path / "x")}) == tmp_path / "x"
persistent = tmp_path / "data"
persistent.mkdir()
assert resolve_root({}, persistent=persistent) == persistent / "unique"
assert resolve_root({"MOUH_DATA_DIR": str(tmp_path / "h")}, persistent=tmp_path / "absent") == tmp_path / "h" / "unique"
def test_store_layout_inventory_and_adapter(tmp_path):
store = LocalStore(tmp_path / "u", persistent_root=tmp_path / "data")
assert store.data_dir.is_dir() and store.failures_dir.is_dir() and not store.persistent
(store.data_dir / "a.jsonl").write_text('{"x": 1}\n\n{"x": 2}\n')
(store.failures_dir / "f.jsonl").write_text('{"y": 1}\n')
assert store.inventory() == {"data": {"fichiers": 1, "lignes": 2}, "failures": {"fichiers": 1, "lignes": 1}}
assert not store.has_adapter() and store.adapter_stamp() is None
store.adapter_dir.mkdir()
(store.adapter_dir / "adapter_config.json").write_text("{}")
assert store.has_adapter() and store.adapter_stamp() is not None
assert LocalStore(tmp_path / "data" / "unique", persistent_root=tmp_path / "data").persistent
# ================================================================= dépôt de JSONL vérifiés
def test_a_simple_task_with_a_real_bug_feeds_sft_grpo_and_dpo(tmp_path):
from unique_ext.training.pairs import preference_pairs
store = LocalStore(tmp_path / "u")
instr = instruction_for("train")
rep = ingest_jsonl(line(instruction=instr, tests=TESTS, solution=GOOD, level="extrême", rejected=BUGGY) + "\n",
"lot1.jsonl", store.data_dir, store.failures_dir)
assert (rep.sessions, rep.failures, sum(rep.rejected.values())) == (1, 1, 0) and len(rep.written) == 2
rec = json.loads(next(store.data_dir.glob("upload-*.jsonl")).read_text())
assert rec["certified"] and rec["generator"] == "upload:lot1" and rec["attempts"][0]["level"] == "extrême"
assert [a["certified"] for a in rec["attempts"]] == [False, True] and "-2" in rec["attempts"][0]["diagnostics"][0]
train, _heldout, stats = JOB.build_examples([rec]) # SFT : génération + réparation, en tête de file
assert stats["gen"] == 1 and stats["fix"] == 1 and train[0]["tier"] == 0
assert JOB._train_tasks([rec], [])[0]["level"] == "extrême" # curriculum GRPO
job_store = JOB.Store(local_out=str(store.root), adapter_repo="local", bucket="b", token=None)
fails = JOB.load_failures(job_store, "local", None, 10) # le job lit l'échec tel quel
assert len(fails) == 1 and fails[0]["task_hash"] == rec["task_hash"]
ctx = SimpleNamespace(sessions=[rec], frozen=set(), render_files=render_files, chat_messages=chat_messages,
load_failures=lambda n: fails)
pairs = preference_pairs(ctx, 10) # DPO : logique native, data/ + failures/
assert len(pairs) == 1 and "m = 0" in pairs[0]["rejected"][0]["content"]
def test_nothing_enters_without_passing_the_sandbox(tmp_path):
store = LocalStore(tmp_path / "u")
instr = instruction_for("train")
session_bad = {"schema": "mouh-session/1", "certified": True, "instruction": instr,
"tests": {"test_solution.py": TESTS}, "final_files": {"solution.py": BUGGY}, "attempts": []}
failure_ok = {"task_hash": "t", "instruction": instr, "tests": {"test_solution.py": TESTS},
"files": {"solution.py": GOOD}, "pytest_output": "E"}
rows = [
line(instruction=instr, tests=TESTS, solution=BUGGY), # solution fausse
line(instruction=instr, tests=TESTS.replace("from solution import maximum", "import math"), solution=GOOD),
line(prompt="p", chosen="c", rejected="r"), # invérifiable
"{pas du json",
json.dumps(session_bad),
json.dumps(failure_ok), # « échec » qui passe
line(instruction=instr, tests=TESTS, solution=GOOD, rejected=GOOD.replace("m = xs[0]", "m = xs[0] # idem")),
line(instruction=instr, tests=TESTS, solution=GOOD), # doublon de la précédente
]
rep = ingest_jsonl("\n".join(rows), "mixte.jsonl", store.data_dir, store.failures_dir)
assert rep.sessions == 1 and rep.failures == 0
assert rep.rejected == {"la solution échoue à ses tests": 2, "les tests n'importent pas solution": 1,
"paire prompt/chosen/rejected sans tests (invérifiable)": 1, "JSON illisible": 1,
"l'échec annoncé passe les tests": 1, "doublon dans le fichier": 1}
assert "ligne 1" in rep.markdown() and "refusées" in rep.markdown()
# sans réexécution, une session déjà certifiée par sa source est reprise telle quelle
rep2 = ingest_jsonl(json.dumps(session_bad), "confiance.jsonl", store.data_dir, store.failures_dir, verify=False)
assert rep2.sessions == 1
def test_heldout_tasks_are_kept_but_reported(tmp_path):
store = LocalStore(tmp_path / "u")
rep = ingest_jsonl(line(instruction=instruction_for("heldout"), tests=TESTS, solution=GOOD), "h.jsonl",
store.data_dir, store.failures_dir)
assert rep.sessions == 1 and rep.heldout == 1 and "examen" in rep.markdown()
# ================================================================= entraînement local
class FakeProc:
def __init__(self, cmd, **kw):
self.cmd, self.kw, self.code = cmd, kw, None
self.released = threading.Event()
self.terminated = False
def poll(self):
return self.code
def wait(self, timeout=None):
if not self.released.wait(timeout if timeout is not None else 10):
raise AssertionError("processus jamais terminé")
return self.code
def finish(self, code=0):
self.code = code
self.released.set()
def terminate(self):
self.terminated = True
self.finish(-15)
def kill(self):
self.finish(-9)
def wait_for(cond, timeout=5.0):
end = time.time() + timeout
while time.time() < end:
if cond():
return True
time.sleep(0.01)
return False
def test_trainer_command_runs_the_job_on_local_data_without_any_hub_call(tmp_path):
store = LocalStore(tmp_path / "u")
t = LocalTrainer(store, env={"UNIQUE_PHASES": "kto:100"})
cmd = t.command()
for flag, value in (("--local-out", str(store.root)), ("--local-data", str(store.data_dir)),
("--space-id", ""), ("--daily-gpu-hours", "24"), ("--benchmark", "0")):
assert cmd[cmd.index(flag) + 1] == value
env = t.child_env()
assert env["UNIQUE_EXT_PATH"] == str(ROOT) and env["UNIQUE_PHASES"] == "kto:100"
t.dpo = True
assert t.child_env()["UNIQUE_PHASES"] == "kto:100,dpo:200"
assert "--benchmark" not in LocalTrainer(store, env={"UNIQUE_BENCHMARK": "1"}).command()
def test_trainer_refuses_without_gpu_and_runs_in_a_loop_with_one(tmp_path):
store = LocalStore(tmp_path / "u")
procs = []
def popen(cmd, **kw):
procs.append(FakeProc(cmd, **kw))
return procs[-1]
no_gpu = LocalTrainer(store, gpu=lambda: {"ok": False, "detail": "aucun GPU"}, popen=popen)
assert "impossible" in no_gpu.start() and procs == []
t = LocalTrainer(store, gpu=lambda: {"ok": True, "detail": "NVIDIA A10G"}, popen=popen, restart_delay_s=0.01,
quick_exit_s=0)
assert "A10G" in t.start() and wait_for(lambda: t.running)
assert procs[0].kw["env"]["UNIQUE_EXT_PATH"] == str(ROOT) and procs[0].kw["cwd"] == str(ROOT)
assert "déjà" in t.start()
procs[0].finish(0) # fin normale d'un passage : relance automatique
assert wait_for(lambda: len(procs) == 2 and t.running)
assert "arrêté" in t.stop(wait_s=1) and procs[1].terminated and not t.running
assert wait_for(lambda: not t._thread.is_alive()) and len(procs) == 2 # plus de relance après l'arrêt
assert "passage 2" in t.log_tail()
def test_gpu_probe_never_touches_cuda_in_the_interface():
def runner(answers):
def run(cmd, **kw):
out = answers.pop(0)
if isinstance(out, Exception):
raise out
return SimpleNamespace(returncode=out[0], stdout=out[1])
return run
assert gpu_status(runner([(0, "NVIDIA A10G, 23028 MiB\n")]))["ok"]
assert gpu_status(runner([FileNotFoundError("nvidia-smi"), (0, "NVIDIA A10G\n")]))["detail"] == "NVIDIA A10G"
assert not gpu_status(runner([FileNotFoundError("nvidia-smi"), (0, "\n")]))["ok"]
def test_old_hub_jobs_are_suspended_and_running_ones_cancelled():
class Api:
def __init__(self):
self.suspended, self.cancelled = [], []
def list_scheduled_jobs(self, labels):
if labels["app"] == "unique-train":
return [SimpleNamespace(id="s1", suspend=False)]
if labels["app"] == "unique-teacher":
return [SimpleNamespace(id="s2", suspend=True)] # déjà suspendu : on n'y touche pas
return []
def suspend_scheduled_job(self, scheduled_job_id):
self.suspended.append(scheduled_job_id)
def list_jobs(self, status, labels):
return [SimpleNamespace(id="j1")] if labels["app"] == "unique-train" else []
def cancel_job(self, job_id):
self.cancelled.append(job_id)
api = Api()
assert suspend_remote_jobs(None, api=api) == ["unique-train suspendu"] and api.cancelled == []
api = Api()
done = suspend_remote_jobs(None, api=api, cancel_running=True)
assert api.suspended == ["s1"] and api.cancelled == ["j1"] and "unique-train en cours annulé" in done
assert suspend_remote_jobs(None) == [] # sans jeton : rien n'est tenté
def test_one_time_import_brings_back_data_adapter_and_frozen_exam(tmp_path):
store = LocalStore(tmp_path / "u")
def download(repo_id, local_dir, allow_patterns, token=None, repo_type=None):
root = Path(local_dir)
if repo_type == "dataset":
(root / "data").mkdir(parents=True, exist_ok=True)
(root / "data" / "teacher-1.jsonl").write_text("{}\n")
(root / "failures").mkdir(parents=True, exist_ok=True)
(root / "failures" / "r1.jsonl").write_text("{}\n")
else:
root.mkdir(parents=True, exist_ok=True)
for name in ("adapter_config.json", "adapter_model.safetensors"):
(root / name).write_text("x")
(root / "state.json").write_text(json.dumps({"heldout_tasks": [1, 2], "best": {"passed": 85}}))
done = import_from_hub(store, "me/data", "me/adapter", None, download=download)
assert len(done) == 3 and store.has_adapter() and store.read_json("state.json")["best"]["passed"] == 85
assert store.inventory()["failures"]["lignes"] == 1 and not (store.root / "import-adapter").exists()
# ================================================================= tableau de bord
STATE = {"base": {"passed": 80, "n": 100}, "best": {"passed": 85, "n": 100, "source": "sft r c1"}, "history": [
{"at": "2026-11-04T22:39:49", "cycle": 1, "phase": "sft", "train": {"loss": 0.1822},
"candidate": {"rate": 0.85}, "best_before": {"rate": 0.8}, "base": {"rate": 0.8}, "promoted": True},
{"at": "2026-11-04T23:44:53", "cycle": 1, "phase": "grpo", "train": {"loss": 0.0158},
"candidate": {"rate": 0.85}, "best_before": {"rate": 0.85}, "base": {"rate": 0.8}, "promoted": False},
{"at": "2026-11-05T00:46:23", "cycle": 2, "phase": "sft", "train": {"loss": 0.1501},
"candidate": {"rate": 0.81}, "best_before": {"rate": 0.85}, "base": {"rate": 0.8}, "promoted": False}]}
def test_dashboard_history_curves_and_best_losses(tmp_path):
rows = dashboard.cycle_rows(STATE)
assert [r["candidat"] for r in rows] == [85.0, 85.0, 81.0] and rows[0]["promu"]
assert dashboard.best_losses(rows) == {"sft": 0.1501, "grpo": 0.0158}
pf, lf = dashboard.pass_frame(rows), dashboard.loss_frame(rows)
assert list(pf.columns) == ["n", "taux", "série"] and len(pf) == 9 and len(lf) == 3
assert dashboard.table(rows)[0][0] == "2026-11-05 00:46" and dashboard.table(rows)[-1][-1] == "oui"
store = LocalStore(tmp_path / "u", persistent_root=tmp_path / "data")
trainer = LocalTrainer(store, gpu=lambda: {"ok": False, "detail": "aucun GPU"})
md = dashboard.status_markdown(store=store, trainer=trainer, gpu={"detail": "aucun GPU"}, state=STATE,
progress={"phase": "sft", "step": 3, "total": 125, "status": "SFT",
"metrics": {"loss": 0.2}}, inventory=store.inventory())
assert "non persistant" in md and "base 80" in md and "meilleur 85" in md and "historique" in md and "sft 0.1501" in md
assert "phase sft 3/125" in md
assert dashboard.cycle_rows(None, {"history": [{"at": "x", "candidate": 0.84, "best": 0.85, "base": 0.8}]})[0][
"candidat"] == 84.0
def test_interface_sessions_stay_local(tmp_path):
from mouh.learning import recorder
rec = recorder.use_folder(tmp_path / "u")
try:
rec.record({"schema": "mouh-session/1"})
assert not rec.remote and len(list((tmp_path / "u" / "data").glob("sessions-*.jsonl"))) == 1
assert recorder.get_recorder() is rec
finally:
recorder._RECORDER = None
# ================================================================= interface
@pytest.fixture
def local_app(tmp_path, monkeypatch):
monkeypatch.setenv("UNIQUE_LOCAL_ROOT", str(tmp_path / "u"))
monkeypatch.setenv("UNIQUE_ADMIN_KEY", "secret")
import app
importlib.reload(app)
yield app
monkeypatch.delenv("UNIQUE_LOCAL_ROOT")
importlib.reload(app)
def test_interface_has_no_online_inference_and_keeps_the_verification_api(local_app):
app = local_app
assert "Modèle M.O.U.H." not in app.UI_CHOICES and app.UI_CHOICES[0] == "Vérification seule (code fourni)"
named = app.demo.get_api_info()["named_endpoints"]
assert [p["parameter_name"] for p in named["/run"]["parameters"]] == [
"instruction", "code", "tests", "proofs", "max_attempts", "generator", "record"]
src = (ROOT / "app.py").read_text(encoding="utf-8")
assert "ddgs" not in src and "vllm" not in src.lower() and "autopilot" not in src
assert "ddgs" not in (ROOT / "requirements.txt").read_text()
status, pass_df, loss_df, table, console = app.refresh()
assert "Mode local" in status and list(pass_df.columns) == ["n", "taux", "série"]
def test_interface_commands_need_the_key_and_uploads_are_verified(local_app, tmp_path):
app = local_app
assert app.start_cmd("mauvaise", False) == app.REFUSED and app.upload_cmd([], True, "") == app.REFUSED
f = tmp_path / "lot.jsonl"
f.write_text(line(instruction=instruction_for("train"), tests=TESTS, solution=GOOD) + "\n")
msg = app.upload_cmd([str(f)], True, "secret")
assert "1 session(s) certifiée(s)" in msg and app.STORE.inventory()["data"]["lignes"] == 1
app.TRAINER.proc = FakeProc(["x"]) # entraînement en cours : le GPU n'est pas partagé
out = list(app.run("t", GOOD, TESTS, "", 1, app.UI_CHOICES[1]))
assert "GPU occupé" in out[0][1]
app.TRAINER.proc.finish(0)
def test_a_passage_that_ends_at_once_is_not_relaunched_every_30_seconds(tmp_path):
store = LocalStore(tmp_path / "u")
procs = []
def popen(cmd, stdout=None, **kw):
stdout.write(b"[16:08:59] Fetching 4 files: 25%|##5 | 1/4 [00:09<00:28, 9.39s/it]\n"
b"[16:09:00] pas assez de sessions certifiees : 0\n")
procs.append(FakeProc(cmd, **kw))
return procs[-1]
t = LocalTrainer(store, gpu=lambda: {"ok": True, "detail": "NVIDIA A10G"}, popen=popen, restart_delay_s=0.01,
idle_delay_s=30)
t.start()
assert wait_for(lambda: t.running)
procs[0].finish(0)
assert wait_for(lambda: "passage très court" in t.log_tail())
time.sleep(0.2)
assert len(procs) == 1 and t.last_reason == "[16:09:00] pas assez de sessions certifiees : 0"
assert "pas assez de sessions" in t.log_tail()
t.stop(wait_s=1)
assert wait_for(lambda: not t._thread.is_alive())
# -- v4.1.2 : le cache du modèle reste hors du stockage monté, la RAM est mesurée ------------------------------
def test_model_cache_stays_off_the_mounted_storage_by_default():
root = Path("/data/unique")
assert hf_home_for(root, {}) is None
assert hf_home_for(root, {"UNIQUE_HF_ON_DATA": "0"}) is None
assert hf_home_for(root, {"UNIQUE_HF_ON_DATA": "1"}) == Path("/data/unique/hf")
assert hf_home_for(Path("/home/user/data/unique"), {"UNIQUE_HF_ON_DATA": "1"}) is None # dossier non monté
def test_app_does_not_point_hf_home_at_the_mounted_storage(monkeypatch):
source = (ROOT / "app.py").read_text(encoding="utf-8")
assert 'startswith("/data/")' not in source and "hf_home_for(ROOT)" in source
def test_ram_report_reads_process_and_container_memory(tmp_path):
(tmp_path / "proc/self").mkdir(parents=True)
(tmp_path / "proc/self/status").write_text("Name:\tpython\nVmRSS:\t 3145728 kB\n")
cg = tmp_path / "sys/fs/cgroup"
cg.mkdir(parents=True)
(cg / "memory.current").write_text(str(14 * 2**30))
(cg / "memory.stat").write_text(f"anon {2 * 2**30}\nfile {10 * 2**30}\n")
assert JOB.ram_report(str(tmp_path)) == "processus 3,0 Go · conteneur 14,0 Go (dont cache de fichiers 10,0 Go)"
def test_ram_report_never_raises_when_nothing_is_readable(tmp_path):
assert JOB.ram_report(str(tmp_path)) == "mesure indisponible"
assert isinstance(JOB.ram_report(), str) # sur la vraie machine : ne lève rien
def test_dashboard_uses_the_current_frozen_exam(tmp_path):
store = LocalStore(tmp_path / "u", persistent_root=tmp_path / "data")
trainer = LocalTrainer(store, gpu=lambda: {"ok": False, "detail": "aucun GPU"})
text = dashboard.status_markdown(store=store, trainer=trainer, gpu={}, state=STATE, progress=None,
inventory=store.inventory(), exam_state={"gele": True, "taches": 84, "sig": "current",
"base": {"passed": 59}, "service": {"passed": 58}})
assert "84 tâches" in text and "59/84 (70.24 %)" in text and "58/84 (69.05 %)" in text
assert "100 tâches" not in text