Spaces:
Sleeping
Sleeping
Update jobs/train_unique.py
Browse files- jobs/train_unique.py +95 -44
jobs/train_unique.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
# /// script
|
| 2 |
# requires-python = ">=3.10"
|
| 3 |
# dependencies = [
|
| 4 |
# "torch>=2.4",
|
|
@@ -576,12 +576,32 @@ def curriculum_select(tasks: list[dict], stats: dict, n: int, rng: random.Random
|
|
| 576 |
# ------------------------------------------------------------------------------------------------
|
| 577 |
# Phases
|
| 578 |
# ------------------------------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 579 |
def _precision():
|
| 580 |
import torch
|
| 581 |
|
| 582 |
if not torch.cuda.is_available():
|
| 583 |
return {"fp16": False, "bf16": False}
|
| 584 |
-
|
|
|
|
|
|
|
|
|
|
| 585 |
|
| 586 |
|
| 587 |
def load_policy(a, tok, token, adapter: str | None):
|
|
@@ -604,6 +624,8 @@ def run_sft(a, model, tok, examples: list[dict], progress: Progress, out_dir: Pa
|
|
| 604 |
from datasets import Dataset
|
| 605 |
from trl import SFTConfig, SFTTrainer
|
| 606 |
|
|
|
|
|
|
|
| 607 |
rows = [{"prompt": e["messages"], "completion": [{"role": "assistant", "content": render_files(e["target"])}]}
|
| 608 |
for e in examples]
|
| 609 |
cfg = SFTConfig(output_dir=str(out_dir), num_train_epochs=1, per_device_train_batch_size=a.batch_size,
|
|
@@ -621,6 +643,8 @@ def run_grpo(a, model, tok, tasks: list[dict], progress: Progress, out_dir: Path
|
|
| 621 |
from datasets import Dataset
|
| 622 |
from trl import GRPOConfig, GRPOTrainer
|
| 623 |
|
|
|
|
|
|
|
| 624 |
rows = [{"prompt": chat_messages(t["instruction"], t["tests"]), "tests": json.dumps(t["tests"]),
|
| 625 |
"task_hash": t["task_hash"], "instruction": t["instruction"]} for t in tasks]
|
| 626 |
for f in repairs or []: # réflexion fondée sur la vraie trace pytest : corriger un échec réel
|
|
@@ -734,7 +758,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
| 734 |
ap.add_argument("--progress-bucket", default=E("UNIQUE_PROGRESS_BUCKET", "RDS777/unique-progress"))
|
| 735 |
ap.add_argument("--base-model", default=E("UNIQUE_BASE_MODEL", "Qwen/Qwen2.5-Coder-0.5B-Instruct"))
|
| 736 |
ap.add_argument("--space-id", default=E("MOUH_SPACE_ID", "RDS777/TARGET_SPACE"), help="vide = ne pas redémarrer")
|
| 737 |
-
ap.add_argument("--max-hours", type=float, default=float(E("UNIQUE_RUN_HOURS", "
|
| 738 |
help="durée d'un passage ; le passage planifié suivant reprend la boucle")
|
| 739 |
ap.add_argument("--daily-gpu-hours", type=float, default=float(E("UNIQUE_MAX_GPU_HOURS_PER_DAY", "24")))
|
| 740 |
ap.add_argument("--cycles", type=int, default=int(E("UNIQUE_CYCLES", "0")), help="0 = jusqu'à la limite de temps")
|
|
@@ -774,6 +798,15 @@ def main(argv: list[str] | None = None) -> int:
|
|
| 774 |
return 2
|
| 775 |
store = Store(local_out=a.local_out, adapter_repo=a.adapter_repo, bucket=a.progress_bucket, token=token)
|
| 776 |
run_id = _now().strftime("%Y%m%dT%H%M%SZ")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 777 |
progress = Progress(store, run_id=run_id, max_hours=a.max_hours, daily_budget_h=a.daily_gpu_hours,
|
| 778 |
min_interval_s=a.progress_every)
|
| 779 |
if progress.over_budget():
|
|
@@ -847,48 +880,66 @@ def main(argv: list[str] | None = None) -> int:
|
|
| 847 |
if progress.elapsed_h() >= a.max_hours or progress.over_budget():
|
| 848 |
stop = True
|
| 849 |
break
|
| 850 |
-
|
| 851 |
-
|
| 852 |
-
|
| 853 |
-
|
| 854 |
-
|
| 855 |
-
|
| 856 |
-
|
| 857 |
-
|
| 858 |
-
|
| 859 |
-
|
| 860 |
-
|
| 861 |
-
rng = random.Random(f"{a.seed}:{cycle}:{run_id}")
|
| 862 |
-
tasks = _train_tasks(sessions, frozen)
|
| 863 |
-
want = max(a.grpo_batch, size * a.grpo_batch // a.grpo_generations)
|
| 864 |
-
n_rep = int(want * a.repair_fraction) if a.repair_fraction > 0 else 0
|
| 865 |
-
stats_by_task = state.get("task_stats", {})
|
| 866 |
-
if a.curriculum:
|
| 867 |
-
tasks = curriculum_select(tasks, stats_by_task, want - n_rep, rng)
|
| 868 |
else:
|
| 869 |
-
rng.
|
| 870 |
-
tasks =
|
| 871 |
-
|
| 872 |
-
|
| 873 |
-
|
| 874 |
-
|
| 875 |
-
|
| 876 |
-
|
| 877 |
-
|
| 878 |
-
|
| 879 |
-
|
| 880 |
-
|
| 881 |
-
|
| 882 |
-
|
| 883 |
-
|
| 884 |
-
|
| 885 |
-
|
| 886 |
-
|
| 887 |
-
|
| 888 |
-
|
| 889 |
-
|
| 890 |
-
|
| 891 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 892 |
info["seconds"] = round(time.time() - t0, 1)
|
| 893 |
cand = evaluate(model, tok, heldout, a, progress, f"candidat {kind}")
|
| 894 |
promoted = cand["passed"] > best["passed"]
|
|
|
|
| 1 |
+
# /// script
|
| 2 |
# requires-python = ">=3.10"
|
| 3 |
# dependencies = [
|
| 4 |
# "torch>=2.4",
|
|
|
|
| 576 |
# ------------------------------------------------------------------------------------------------
|
| 577 |
# Phases
|
| 578 |
# ------------------------------------------------------------------------------------------------
|
| 579 |
+
def _gpu_compat() -> None:
|
| 580 |
+
"""GPU antérieurs à Ampere (T4 = capacité 7.5) : le noyau « fused LM head » de TRL fait un produit matriciel
|
| 581 |
+
16 bits -> 32 bits que ces GPU refusent (« only supported for compute capability 8.0 or higher »).
|
| 582 |
+
On le force sur son chemin de repli en fp32, un peu plus lent mais compatible."""
|
| 583 |
+
import torch
|
| 584 |
+
|
| 585 |
+
if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] >= 8:
|
| 586 |
+
return
|
| 587 |
+
try:
|
| 588 |
+
from trl.kernels import chunked_logprob
|
| 589 |
+
|
| 590 |
+
chunked_logprob._MM_OUT_DTYPE = False
|
| 591 |
+
log("GPU < Ampere : produit matriciel 16->32 bits de TRL désactivé (repli fp32)")
|
| 592 |
+
except Exception as exc: # noqa: BLE001
|
| 593 |
+
log(f"compatibilité GPU : réglage TRL impossible ({exc})")
|
| 594 |
+
|
| 595 |
+
|
| 596 |
def _precision():
|
| 597 |
import torch
|
| 598 |
|
| 599 |
if not torch.cuda.is_available():
|
| 600 |
return {"fp16": False, "bf16": False}
|
| 601 |
+
# bf16 natif seulement à partir d'Ampere (capacité 8.x : L4, A10G, A100). Sur T4 (7.5), PyTorch annonce un bf16
|
| 602 |
+
# « émulé » que Transformers refuse : on utilise alors fp16.
|
| 603 |
+
major, _ = torch.cuda.get_device_capability()
|
| 604 |
+
return {"bf16": True, "fp16": False} if major >= 8 else {"fp16": True, "bf16": False}
|
| 605 |
|
| 606 |
|
| 607 |
def load_policy(a, tok, token, adapter: str | None):
|
|
|
|
| 624 |
from datasets import Dataset
|
| 625 |
from trl import SFTConfig, SFTTrainer
|
| 626 |
|
| 627 |
+
_gpu_compat()
|
| 628 |
+
|
| 629 |
rows = [{"prompt": e["messages"], "completion": [{"role": "assistant", "content": render_files(e["target"])}]}
|
| 630 |
for e in examples]
|
| 631 |
cfg = SFTConfig(output_dir=str(out_dir), num_train_epochs=1, per_device_train_batch_size=a.batch_size,
|
|
|
|
| 643 |
from datasets import Dataset
|
| 644 |
from trl import GRPOConfig, GRPOTrainer
|
| 645 |
|
| 646 |
+
_gpu_compat()
|
| 647 |
+
|
| 648 |
rows = [{"prompt": chat_messages(t["instruction"], t["tests"]), "tests": json.dumps(t["tests"]),
|
| 649 |
"task_hash": t["task_hash"], "instruction": t["instruction"]} for t in tasks]
|
| 650 |
for f in repairs or []: # réflexion fondée sur la vraie trace pytest : corriger un échec réel
|
|
|
|
| 758 |
ap.add_argument("--progress-bucket", default=E("UNIQUE_PROGRESS_BUCKET", "RDS777/unique-progress"))
|
| 759 |
ap.add_argument("--base-model", default=E("UNIQUE_BASE_MODEL", "Qwen/Qwen2.5-Coder-0.5B-Instruct"))
|
| 760 |
ap.add_argument("--space-id", default=E("MOUH_SPACE_ID", "RDS777/TARGET_SPACE"), help="vide = ne pas redémarrer")
|
| 761 |
+
ap.add_argument("--max-hours", type=float, default=float(E("UNIQUE_RUN_HOURS", "11.5")),
|
| 762 |
help="durée d'un passage ; le passage planifié suivant reprend la boucle")
|
| 763 |
ap.add_argument("--daily-gpu-hours", type=float, default=float(E("UNIQUE_MAX_GPU_HOURS_PER_DAY", "24")))
|
| 764 |
ap.add_argument("--cycles", type=int, default=int(E("UNIQUE_CYCLES", "0")), help="0 = jusqu'à la limite de temps")
|
|
|
|
| 798 |
return 2
|
| 799 |
store = Store(local_out=a.local_out, adapter_repo=a.adapter_repo, bucket=a.progress_bucket, token=token)
|
| 800 |
run_id = _now().strftime("%Y%m%dT%H%M%SZ")
|
| 801 |
+
prev = store.read_progress() or {}
|
| 802 |
+
try:
|
| 803 |
+
age = (_now() - datetime.fromisoformat(prev["updated_at"])).total_seconds()
|
| 804 |
+
except Exception: # noqa: BLE001
|
| 805 |
+
age = 1e9
|
| 806 |
+
if prev.get("phase") not in (None, "fin", "pause", "attente") and age < 600 and a.local_out is None:
|
| 807 |
+
# un autre passage écrit encore sa progression (ex. job recréé pendant un passage) : jamais deux GPU à la fois
|
| 808 |
+
log(f"un passage est déjà en cours ({prev.get('run_id')}, mis à jour il y a {int(age)} s) : arrêt.")
|
| 809 |
+
return 0
|
| 810 |
progress = Progress(store, run_id=run_id, max_hours=a.max_hours, daily_budget_h=a.daily_gpu_hours,
|
| 811 |
min_interval_s=a.progress_every)
|
| 812 |
if progress.over_budget():
|
|
|
|
| 880 |
if progress.elapsed_h() >= a.max_hours or progress.over_budget():
|
| 881 |
stop = True
|
| 882 |
break
|
| 883 |
+
try:
|
| 884 |
+
model = load_policy(a, tok, token, store.adapter_source())
|
| 885 |
+
t0 = time.time()
|
| 886 |
+
if kind == "sft":
|
| 887 |
+
order = sorted(range(len(train)), key=lambda i: hashlib.sha256(
|
| 888 |
+
f"{a.seed}:{_sha(train[i]['target'])}:{train[i]['kind']}:{i}".encode()).hexdigest())
|
| 889 |
+
cur = state.get("sft_cursor", 0) % max(1, len(order))
|
| 890 |
+
idx = (order[cur:] + order[:cur])[: min(size, len(order))]
|
| 891 |
+
state["sft_cursor"] = cur + len(idx)
|
| 892 |
+
progress.phase("sft", 1, f"cycle {cycle} : SFT sur {len(idx)} exemples")
|
| 893 |
+
info = run_sft(a, model, tok, [train[i] for i in idx], progress, work / "sft")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 894 |
else:
|
| 895 |
+
rng = random.Random(f"{a.seed}:{cycle}:{run_id}")
|
| 896 |
+
tasks = _train_tasks(sessions, frozen)
|
| 897 |
+
want = max(a.grpo_batch, size * a.grpo_batch // a.grpo_generations)
|
| 898 |
+
n_rep = int(want * a.repair_fraction) if a.repair_fraction > 0 else 0
|
| 899 |
+
stats_by_task = state.get("task_stats", {})
|
| 900 |
+
if a.curriculum:
|
| 901 |
+
tasks = curriculum_select(tasks, stats_by_task, want - n_rep, rng)
|
| 902 |
+
else:
|
| 903 |
+
rng.shuffle(tasks)
|
| 904 |
+
tasks = tasks[: want - n_rep]
|
| 905 |
+
banned = set(frozen)
|
| 906 |
+
repairs = [f for f in load_failures(store, a.dataset_repo, token, 2000) if f["task_hash"] not in banned]
|
| 907 |
+
rng.shuffle(repairs)
|
| 908 |
+
repairs = repairs[:n_rep]
|
| 909 |
+
rollout = RolloutLog()
|
| 910 |
+
progress.phase("grpo", size, f"cycle {cycle} : GRPO (récompense pytest), {len(tasks)} tâches"
|
| 911 |
+
f" + {len(repairs)} réparations d'échecs réels")
|
| 912 |
+
info = run_grpo(a, model, tok, tasks, progress, work / "grpo", repairs=repairs, rollout=rollout)
|
| 913 |
+
merged = dict(stats_by_task)
|
| 914 |
+
for h, (ok_, n_) in rollout.outcomes.items():
|
| 915 |
+
old = merged.get(h, [0, 0])
|
| 916 |
+
merged[h] = [old[0] + ok_, old[1] + n_]
|
| 917 |
+
state["task_stats"] = dict(list(merged.items())[-50000:])
|
| 918 |
+
zero = sum(1 for h, (ok_, n_) in rollout.outcomes.items() if ok_ in (0, n_))
|
| 919 |
+
info["curriculum"] = {"tasks_seen": len(rollout.outcomes),
|
| 920 |
+
"informative": len(rollout.outcomes) - zero, "failures": len(rollout.failures)}
|
| 921 |
+
if a.save_failures:
|
| 922 |
+
try:
|
| 923 |
+
save_failures(store, a.dataset_repo, rollout.failures, f"{run_id}-c{cycle}.jsonl")
|
| 924 |
+
except Exception as exc: # noqa: BLE001
|
| 925 |
+
log(f"sauvegarde des échecs impossible : {exc}")
|
| 926 |
+
except Exception as exc: # noqa: BLE001 — une phase en échec ne doit pas arrêter toute la boucle
|
| 927 |
+
import traceback
|
| 928 |
+
|
| 929 |
+
tb = traceback.format_exc()
|
| 930 |
+
log(f"ÉCHEC de la phase {kind} :\n{tb}")
|
| 931 |
+
failures_in_row = state.get("phase_errors", 0) + 1
|
| 932 |
+
state["phase_errors"] = failures_in_row
|
| 933 |
+
state["last_error"] = {"at": _now().isoformat(), "phase": kind, "error": tb[-1500:]}
|
| 934 |
+
store.write_json("state.json", state, f"erreur {run_id} c{cycle} {kind}")
|
| 935 |
+
progress.update(important=True, status=f"erreur pendant {kind} : {str(exc)[:200]}", last_error=state["last_error"])
|
| 936 |
+
if torch.cuda.is_available():
|
| 937 |
+
torch.cuda.empty_cache()
|
| 938 |
+
if failures_in_row >= 3: # erreur persistante : on s'arrête au lieu de payer du GPU pour rien
|
| 939 |
+
progress.update(important=True, phase="erreur", status=f"arrêt après 3 échecs : {str(exc)[:200]}")
|
| 940 |
+
return 1
|
| 941 |
+
continue
|
| 942 |
+
state["phase_errors"] = 0
|
| 943 |
info["seconds"] = round(time.time() - t0, 1)
|
| 944 |
cand = evaluate(model, tok, heldout, a, progress, f"candidat {kind}")
|
| 945 |
promoted = cand["passed"] > best["passed"]
|