SevdanurGenc's picture
Upload 300 files
9438a57 verified
Raw History Blame Contribute Delete
23 kB
"""Makale deneylerini arayüzden çalıştıran, kesintiye dayanıklı deney yöneticisi.
Bir deney, sıralı adımlardan oluşur: (model × tur sayısı) değerlendirmeleri, ablasyonlar ve hakem doğruluğu
ölçümleri. Durum her adımda ve her öğeden sonra diske yazılır. Uygulama kapanır (bilgisayar uyur, yeniden
başlar) ve açılırsa deney kaldığı öğeden kendiliğinden sürer. Sonuçlar tek bir zip olarak indirilir.
"""
from __future__ import annotations
import io
import json
import re
import subprocess
import sys
import shutil
import threading
import time
import uuid
import zipfile
from pathlib import Path
from typing import Any, Literal, Optional
from pydantic import BaseModel
from ..config import settings
from . import benchmark
from .report import render
from .stats import render_analysis
ROOT = Path(__file__).resolve().parents[3]
ABL_FULL = ["judge", "prune", "restart", "repair", "planfree", "best", "mutfix"]
ABL_QUICK = ["judge", "prune", "restart", "mutfix"]
# Deneme profili: hattın uçtan uca çalıştığını ~5 dakikada gösteren iki küçük program (makale verisi değildir)
SMOKE_ITEMS = ["qb-py-bitcount", "qb-py-flatten"]
SMOKE_MUTANTS = 10
# Tahmini süre için öğe başına varsayılan (ilk öğe bitene kadar kullanılır)
DEFAULT_ITEM_S = 240.0
SMOKE_ITEM_S = 120.0
JUDGE_STEP_S = 900.0
PDF_IN_ZIP = True # sonuç zip'ine raporların PDF'leri de eklenir (tarayıcı varsa)
XMUT_ITEM_S = 30.0 # harici mutasyon aracı, öğe başına tahmini süre
# Model karşılaştırma profili: her model aynı öğelerde, tam döngüyle (en çok 4 tur) ve tekrarla
COMPARE_REPEATS = 3
class Step(BaseModel):
label: str
kind: Literal["eval", "judge", "xmut"] = "eval"
source: str = "" # xmut: ölçülecek değerlendirme adımının etiketi
retry: bool = False # xmut: araç hatasıyla kalan ölçümleri yeniden dene
model: str
max_iterations: int = 4
repeats: int = 1
ablate: list[str] = []
rule_mode: str = "veto"
status: Literal["pending", "running", "done", "failed", "paused", "cancelled"] = "pending"
eval_id: str = ""
started_at: float = 0.0
finished_at: float = 0.0
error: str = ""
out_file: str = ""
class Experiment(BaseModel):
id: str
created_at: float
profile: Literal["tam", "hizli", "deneme", "karsilastirma"]
models: list[str]
status: Literal["running", "paused", "done", "failed", "cancelled"] = "running"
steps: list[Step] = []
max_mutants: int = 30
item_ids: list[str] = [] # boşsa tüm veri seti
finished_at: float = 0.0
error: str = ""
def _slug(m: str) -> str:
return re.sub(r"[^a-zA-Z0-9]+", "-", m).strip("-")
def plan(profile: str, models: list[str]) -> list[Step]:
"""deney-calistir betikleriyle aynı plan: her model için tur 1 ve tur 4, büyük modelde ablasyonlar, hakem.
"deneme": yalnızca ilk model, iki küçük öğe, tam döngü (tur 4 + mutant turu) — ~5 dakikalık hat denemesi."""
def xmut(src: Step) -> Step:
return Step(label=f"xmut-{src.label}", kind="xmut", model=src.model, source=src.label)
if profile == "deneme":
st = Step(label=f"deneme-{_slug(models[0])}-k4", model=models[0], max_iterations=4, repeats=1)
return [st, xmut(st)]
if profile == "karsilastirma":
# Yerel ve bulut modellerini aynı koşullarda karşılaştırma: her model × tüm öğeler × 3 tekrar (k=4)
evs = [Step(label=f"karsilastirma-{_slug(m)}-k4", model=m, max_iterations=4, repeats=COMPARE_REPEATS)
for m in models]
return evs + [xmut(x) for x in evs]
full = profile == "tam"
rep = 3 if full else 1
big = models[-1]
steps = [Step(label=f"deney-{_slug(m)}-k{k}", model=m, max_iterations=k, repeats=rep)
for m in models for k in (1, 4)]
steps += [Step(label=f"deney-abl-{a}", model=big, max_iterations=4, repeats=1, ablate=[a])
for a in (ABL_FULL if full else ABL_QUICK)]
for m in (models if full else [big]):
for mode in (["veto", "hint", "off"] if full else ["veto", "off"]):
steps.append(Step(label=f"hakem-{_slug(m)}-{mode}", kind="judge", model=m, rule_mode=mode))
# Harici mutasyon araçlarıyla çapraz doğrulama (k=4 ana yapılandırmalar; hızlı profilde yalnızca büyük model)
k4 = [x for x in steps if x.kind == "eval" and x.max_iterations == 4 and not x.ablate]
steps += [xmut(x) for x in (k4 if full else [x for x in k4 if x.model == big])]
return steps
LLM_FACTORY = None # yalnızca otomatik testler için (sahte LLM)
_exps: dict[str, Experiment] = {}
_threads: dict[str, threading.Thread] = {}
_pause: dict[str, threading.Event] = {}
_lock = threading.Lock()
def _dir() -> Path:
d = settings.data_dir / "experiments"
d.mkdir(parents=True, exist_ok=True)
return d
def persist(e: Experiment) -> None:
with _lock:
(_dir() / f"{e.id}.json").write_text(e.model_dump_json(indent=2))
def load_persisted() -> None:
for f in _dir().glob("*.json"):
try:
e = Experiment.model_validate_json(f.read_text())
_exps.setdefault(e.id, e)
except Exception: # noqa: BLE001
continue
def get(exp_id: str) -> Optional[Experiment]:
return _exps.get(exp_id)
def all_experiments() -> list[Experiment]:
return sorted(_exps.values(), key=lambda e: e.created_at, reverse=True)
def active() -> Optional[Experiment]:
return next((e for e in all_experiments() if e.status == "running"), None)
def create(profile: str, models: list[str], max_mutants: int = 30, background: bool = True,
item_ids: Optional[list[str]] = None, judge: bool = True) -> Experiment:
if active():
raise RuntimeError("busy")
steps = [s for s in plan(profile, models) if judge or s.kind != "judge"]
if profile == "deneme":
item_ids, max_mutants = item_ids or SMOKE_ITEMS, min(max_mutants, SMOKE_MUTANTS)
e = Experiment(id=uuid.uuid4().hex[:8], created_at=time.time(), profile=profile, models=models,
steps=steps, max_mutants=max_mutants, item_ids=item_ids or [])
_exps[e.id] = e
persist(e)
_launch(e, background)
return e
def _launch(e: Experiment, background: bool = True) -> None:
ev = _pause.setdefault(e.id, threading.Event())
ev.clear()
e.status = "running"
persist(e)
if background:
t = threading.Thread(target=_run, args=(e,), daemon=True)
_threads[e.id] = t
t.start()
else:
_run(e)
def pause(exp_id: str) -> bool:
e = _exps.get(exp_id)
if not e or e.status != "running":
return False
_pause.setdefault(exp_id, threading.Event()).set()
for s in e.steps:
if s.status == "running" and s.eval_id:
benchmark.pause_eval(s.eval_id)
return True
_cancelled: set[str] = set()
def cancel(exp_id: str) -> bool:
"""Deneyi kalıcı olarak durdurur: süren öğe hemen iptal edilir (yarıda kalır), kalan adımlar koşulmaz.
Biten adımların sonuçları korunur ve zip olarak indirilebilir."""
e = _exps.get(exp_id)
if not e or e.status in ("done", "cancelled"):
return False
_cancelled.add(exp_id)
_pause.setdefault(exp_id, threading.Event()).set()
for s in e.steps:
if s.status in ("running", "paused") and s.eval_id:
benchmark.cancel_eval(s.eval_id)
alive = exp_id in _threads and _threads[exp_id].is_alive()
if not alive: # duraklatılmış ya da hatalı deney: iş parçacığı yok, durumu doğrudan kapat
_finish_cancel(e)
return True
def _finish_cancel(e: Experiment) -> None:
for s in e.steps:
if s.status in ("running", "paused", "pending"):
s.status = "cancelled"
e.status, e.finished_at = "cancelled", time.time()
persist(e)
def resume(exp_id: str, background: bool = True) -> Optional[Experiment]:
e = _exps.get(exp_id)
if not e or e.status in ("done", "cancelled"):
return e
if exp_id in _threads and _threads[exp_id].is_alive():
# duraklatma isteği verilmiş ama süren öğe henüz bitmemiş: isteği geri al
_pause.setdefault(exp_id, threading.Event()).clear()
for s in e.steps:
if s.status == "running" and s.eval_id:
benchmark.resume_eval(s.eval_id, llm_factory=LLM_FACTORY)
e.status = "running"
persist(e)
return e
if active() and active().id != exp_id:
raise RuntimeError("busy")
_launch(e, background)
return e
def retryable(e: Experiment) -> dict[str, int]:
"""Yeniden koşulabilecek eksikler: altyapı hatalı öğeler, araç hatalı harici mutasyon ölçümleri, hatalı adımlar."""
infra = 0
for s in e.steps:
ev = benchmark.get_eval(s.eval_id) if s.kind == "eval" and s.eval_id else None
if ev:
infra += sum(1 for i in ev.items if i.status == "failed" and benchmark.is_infra_error(i.error))
tool = 0
for s in e.steps:
f = _dir() / e.id / xmut_file(s) if s.kind == "xmut" else None
if f and f.exists():
try:
tool += sum(1 for r in json.loads(f.read_text(encoding="utf-8"))
if not r.get("ok") and r.get("error") not in ("unsupported", "no_mutants"))
except ValueError:
pass
steps = sum(1 for s in e.steps if s.status == "failed")
return {"items": infra, "xmut": tool, "steps": steps, "total": infra + tool + steps}
def retry_failed(exp_id: str, background: bool = True) -> Optional[Experiment]:
"""Bitmiş (ya da hatayla duran) bir deneyde eksikleri tamamlar: altyapı hatasıyla düşen öğeleri ve araç hatasıyla
kalan harici mutasyon ölçümlerini yeniden koşar. Modelin kendi başarısızlıkları (zaman aşımı vb.) korunur."""
e = _exps.get(exp_id)
if not e or e.status in ("running", "cancelled") or (exp_id in _threads and _threads[exp_id].is_alive()):
return None
if active() and active().id != exp_id:
raise RuntimeError("busy")
reopened = False
for s in e.steps:
if s.kind == "eval" and s.eval_id and benchmark.reopen_infra_failures(s.eval_id):
s.status, s.finished_at, reopened = "pending", 0.0, True
for s in e.steps:
if s.kind == "xmut":
f = _dir() / e.id / xmut_file(s)
src = next((x for x in e.steps if x.label == s.source), None)
bad = False
if f.exists():
try:
bad = any(not r.get("ok") and r.get("error") not in ("unsupported", "no_mutants")
for r in json.loads(f.read_text(encoding="utf-8")))
except ValueError:
bad = True
if bad or (src is not None and src.status != "done"):
s.status, s.retry, reopened = "pending", bad, True
elif s.status == "failed":
s.status, reopened = "pending", True
if not reopened:
return e
e.status, e.error, e.finished_at = "paused", "", 0.0
persist(e)
_launch(e, background)
return e
def resume_interrupted() -> None:
"""Uygulama açılışında: kapanırken 'running' kalan deneyi (ve bağımsız değerlendirmeleri) sürdür."""
running = next((e for e in all_experiments() if e.status == "running"), None)
mine = {s.eval_id for s in running.steps} if running else set()
for ev in benchmark.interrupted(): # bağımsız değerlendirmeler: arayüzden "Sürdür" ile devam edilir
if ev.id not in mine:
ev.status = "paused"
benchmark.persist(ev)
if running:
_launch(running)
def _run_eval_step(e: Experiment, s: Step) -> None:
ev = benchmark.get_eval(s.eval_id) if s.eval_id else None
if ev is None:
cfg = {"max_iterations": s.max_iterations, "max_mutants": e.max_mutants, "coverage_target": 85,
"temperature": 0.2, "run_mutation": True, "ablate": s.ablate}
ev = benchmark.run_eval(s.model, cfg, item_ids=e.item_ids or None, split="all", repeats=s.repeats,
label=s.label, background=True, llm_factory=LLM_FACTORY)
s.eval_id = ev.id
persist(e)
elif ev.status != "done":
benchmark.resume_eval(ev.id, llm_factory=LLM_FACTORY)
stop = _pause[e.id]
while ev.status == "running":
if e.id in _cancelled:
benchmark.cancel_eval(ev.id)
elif stop.is_set():
benchmark.pause_eval(ev.id) # süren öğeden sonra durur
if not benchmark.worker_alive(ev.id) and ev.status == "running":
time.sleep(1)
if ev.status == "running": # işçi beklenmedik biçimde öldü
ev.status = "paused"
benchmark.persist(ev)
s.error = "evaluation worker stopped unexpectedly"
break
time.sleep(2)
if ev.status in ("paused", "cancelled"):
s.status = "paused"
return
s.status = "done"
def _run_judge_step(e: Experiment, s: Step) -> None:
out = _dir() / e.id
out.mkdir(parents=True, exist_ok=True)
name = f"hakem_{s.model.replace(':', '-').replace('/', '-')}_{s.rule_mode}.json"
script = ROOT / "scripts" / "judge_accuracy.py"
p = subprocess.Popen([sys.executable, str(script), "--model", s.model, "--rule-mode", s.rule_mode,
"--out-dir", str(out)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
cwd=str(ROOT))
while p.poll() is None:
if e.id in _cancelled:
p.kill()
s.status = "paused"
return
time.sleep(1)
log = p.stdout.read() if p.stdout else ""
if p.returncode != 0 or not (out / name).exists():
s.status, s.error = "failed", log[-800:]
return
s.out_file, s.status = name, "done"
def xmut_file(s: Step) -> str:
return f"xmut_{_slug(s.source)}.json"
def _run_xmut_step(e: Experiment, s: Step) -> None:
"""Değerlendirme adımının son test paketlerini harici mutasyon araçlarıyla yeniden ölçer (kaldığı yerden sürer)."""
from . import xmut
src = next((x for x in e.steps if x.label == s.source), None)
ev = benchmark.get_eval(src.eval_id) if src and src.eval_id else None
if ev is None:
s.status, s.error = "failed", f"source evaluation not found: {s.source}"
return
out = _dir() / e.id
out.mkdir(parents=True, exist_ok=True)
stop = _pause[e.id]
try:
xmut.check_eval(ev, out / xmut_file(s), should_stop=lambda: e.id in _cancelled or stop.is_set(),
retry_failed=s.retry)
s.retry = False
except Exception as ex: # noqa: BLE001
s.status, s.error = "failed", f"{type(ex).__name__}: {ex}"
return
if e.id in _cancelled or stop.is_set():
s.status = "paused"
return
s.out_file, s.status = xmut_file(s), "done"
def _run(e: Experiment) -> None:
stop = _pause[e.id]
try:
for s in e.steps:
if s.status == "done":
continue
if stop.is_set():
break
s.status, s.error = "running", ""
s.started_at = s.started_at or time.time()
persist(e)
if s.kind == "eval":
_run_eval_step(e, s)
elif s.kind == "xmut":
_run_xmut_step(e, s)
else:
_run_judge_step(e, s)
if s.status == "done":
s.finished_at = time.time()
persist(e)
if s.status == "paused":
break
if e.id in _cancelled:
_cancelled.discard(e.id)
_finish_cancel(e)
return
if stop.is_set() or any(s.status == "paused" for s in e.steps):
e.status = "paused"
for s in e.steps:
if s.status == "running":
s.status = "paused"
elif any(s.status == "failed" for s in e.steps):
# diğer adımlar tamamlandı; "Sürdür" yalnızca başarısız adımları yeniden dener
e.status = "failed"
e.error = "; ".join(f"{s.label}: {s.error.strip().splitlines()[-1] if s.error.strip() else 'hata'}"
for s in e.steps if s.status == "failed")[:600]
else:
e.status, e.error = "done", ""
e.finished_at = time.time()
except Exception as ex: # noqa: BLE001
e.status, e.error = "failed", str(ex)
persist(e)
def progress(e: Experiment) -> dict[str, Any]:
"""Öğe-koşusu bazında ilerleme ve kalan süre tahmini."""
n_items = len(e.item_ids) or len(benchmark.load_manifest())
default_item = SMOKE_ITEM_S if e.profile == "deneme" else DEFAULT_ITEM_S
total = done = 0
durations: list[float] = []
steps = []
for s in e.steps:
st = {"label": s.label, "kind": s.kind, "model": s.model, "status": s.status, "ablate": s.ablate,
"max_iterations": s.max_iterations, "repeats": s.repeats, "rule_mode": s.rule_mode,
"eval_id": s.eval_id, "error": s.error, "done": 0, "total": 0, "source": s.source}
if s.kind == "xmut":
f = _dir() / e.id / xmut_file(s)
try:
st["done"] = len(json.loads(f.read_text(encoding="utf-8"))) if f.exists() else 0
except ValueError:
pass
if s.kind == "eval":
ev = benchmark.get_eval(s.eval_id) if s.eval_id else None
st["total"] = len(ev.items) if ev else n_items * s.repeats
if ev:
fin = [i for i in ev.items if i.status not in ("pending", "running")]
st["done"] = len(fin)
durations += [i.duration_s for i in fin if i.duration_s]
st["summary"] = ev.summary.get("overall", {})
total += st["total"]
done += st["done"]
steps.append(st)
per_item = (sum(durations) / len(durations) * 1.25) if durations else default_item # +%25: mutasyon/tohum
judge_left = sum(1 for s in e.steps if s.kind == "judge" and s.status != "done")
xmut_left = sum(s.repeats for s0 in e.steps if s0.kind == "xmut" and s0.status != "done"
for s in e.steps if s.label == s0.source) * _xmut_items(e)
eta = ((total - done) * per_item + judge_left * JUDGE_STEP_S + xmut_left * XMUT_ITEM_S
if e.status in ("running", "paused", "failed") else 0)
return {**e.model_dump(exclude={"steps"}), "steps": steps, "items_done": done, "items_total": total,
"percent": round(100 * done / total, 1) if total else 0.0, "eta_s": round(eta),
"avg_item_s": round(per_item, 1), "retryable": retryable(e),
"judge_done": sum(1 for s in e.steps if s.kind == "judge" and s.status == "done"),
"judge_total": sum(1 for s in e.steps if s.kind == "judge")}
def _xmut_csv(e: Experiment) -> str:
"""Harici araç ölçümleri, öğe başına bir satır (istatistik yazılımında analiz için)."""
import csv
import io
rows = []
for s in e.steps:
f = _dir() / e.id / xmut_file(s) if s.kind == "xmut" else None
if f and f.exists():
try:
rows += [{"model": s.model, "source": s.source, **r} for r in json.loads(f.read_text(encoding="utf-8"))]
except ValueError:
pass
if not rows:
return ""
cols = ["model", "source", "item", "rep", "language", "split", "ours_pre", "ours", "tool", "ok", "score",
"total", "killed", "survived", "timeout", "seconds", "error"]
buf = io.StringIO()
w = csv.DictWriter(buf, fieldnames=cols, extrasaction="ignore")
w.writeheader()
for r in rows:
w.writerow({k: (str(r.get(k, ""))[:200].replace("\n", " ") if k == "error" else r.get(k, "")) for k in cols})
return buf.getvalue()
def _xmut_items(e: Experiment) -> int:
from .xmut import supported
items = benchmark.load_manifest()
if e.item_ids:
items = [i for i in items if i["id"] in e.item_ids]
return sum(1 for i in items if supported(i["language"], i.get("kind", "unit")))
def zip_name(e: Experiment, lang: str = "tr") -> str:
"""İndirme adı: profil + deneyin başlangıç tarihi/saati (ör. deney_sonuclari_tam_2026-10-10_1216.zip)."""
stamp = time.strftime("%Y-%m-%d_%H%M", time.localtime(e.created_at))
return f"{'experiment_results' if lang == 'en' else 'deney_sonuclari'}_{e.profile}_{stamp}.zip"
def results_zip(e: Experiment, lang: str = "tr") -> bytes:
"""Makale için sonuç paketi: renkli HTML raporlar (TR + EN), değerlendirme JSON/MD'leri (TR + EN),
hakem sonuçları ve istatistiksel analiz (TR + EN)."""
from .exp_report import render_experiment
evs = [benchmark.get_eval(s.eval_id) for s in e.steps if s.kind == "eval" and s.eval_id]
evs = [x for x in evs if x is not None]
out = _dir() / e.id
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z:
rep_tr, rep_en = render_experiment(e, out, "tr"), render_experiment(e, out, "en")
z.writestr("deney/RAPOR_tr.html", rep_tr)
z.writestr("deney/REPORT_en.html", rep_en)
if PDF_IN_ZIP:
from ..pdf import html_to_pdf
tmp = out / f"_pdf_{uuid.uuid4().hex[:6]}"
tmp.mkdir(parents=True, exist_ok=True)
try:
(tmp / "RAPOR_tr.html").write_text(rep_tr, encoding="utf-8")
(tmp / "REPORT_en.html").write_text(rep_en, encoding="utf-8")
pdfs, _ = html_to_pdf([tmp / "RAPOR_tr.html", tmp / "REPORT_en.html"])
for pf in pdfs:
z.write(pf, f"deney/{pf.name}")
except Exception: # noqa: BLE001 (PDF motoru yoksa zip yine HTML raporlarla iner)
pass
finally:
shutil.rmtree(tmp, ignore_errors=True)
z.writestr("deney/ANALIZ.md", render_analysis(evs, "tr"))
z.writestr("deney/ANALYSIS.md", render_analysis(evs, "en"))
for ev in evs:
z.writestr(f"deney/{ev.label}.json", ev.model_dump_json(indent=2))
z.writestr(f"deney/tr/{ev.label}.md", render([ev], lang="tr"))
z.writestr(f"deney/en/{ev.label}.md", render([ev], lang="en"))
for s in e.steps:
if s.kind in ("judge", "xmut") and s.out_file and (out / s.out_file).exists():
z.write(out / s.out_file, f"deney/{s.out_file}")
z.writestr("deney/tum_sonuclar.json", json.dumps([x.model_dump() for x in evs], ensure_ascii=False))
z.writestr("deney/deney.json", json.dumps(progress(e), ensure_ascii=False, indent=2))
x = _xmut_csv(e)
if x:
z.writestr("deney/harici_mutasyon.csv", x)
return buf.getvalue()