aidevices's picture
Sync experiment for HF Jobs Docker image
02ef37e verified
Raw History Blame Contribute Delete
18.5 kB
"""Harness entrypoint for the ARChitects ARC Prize 2024 reproduction (SCRUM-80).
HF Jobs runs this as ``python /app/train.py``. It reproduces upstream's
``run_evaluation_Llama-rearc_{without,with}_ttt.py`` but takes its parameters from the
``HYPERPARAMETERS`` env var (JSON) that commons.infra injects, because the upstream scripts
have no CLI arguments — model, thresholds, augmentation and the full 400-task eval set are
all hardcoded there.
The ARChitects' code is vendored unmodified under ``vendor/architects/`` (Apache-2.0,
Copyright 2024 Daniel Franzen and Jan Disselhoff). Only this file is ours.
Dataset: ARC-AGI-1's 400 *public evaluation* tasks, fetched at run time from
github.com/fchollet/ARC-AGI by the vendored ``arc_downloader``.
NOTE ON SCORES: the paper's headline 53.5% is on Kaggle's 100 *private* test tasks,
which are not public. Scores produced here are on the public evaluation split and are
NOT directly comparable to 53.5%.
Hyperparameters (all optional; defaults shown):
mode "without_ttt" | "with_ttt" inference only, or test-time training first
base_model HF model id to load in 4-bit
full_eval true -> all 400 eval tasks; false -> one shard (see subset_*)
subset_splits number of shards to cut the eval set into (40 -> ~10 tasks/shard)
subset_index which shard to run
split_seed shard permutation seed
min_prob DFS probability threshold (upstream: 0.1)
n_guesses submitted candidates per task (upstream: 2)
infer_repeat dataset repeats before augmentation (upstream: 2)
infer_seed augmentation seed (upstream: 10000)
ttt_repeat TTT demo-pair repeats (upstream: 48), only used when mode=with_ttt
"""
from __future__ import annotations
import json
import os
import sys
from pathlib import Path
# Vendored upstream modules import each other by bare name (``from arc_loader import ...``),
# so their directory must be importable directly.
_VENDOR = Path(__file__).resolve().parent / "vendor" / "architects"
sys.path.insert(0, str(_VENDOR))
OUT_DIR = Path("/app/out")
ARC_DATA_PATH = Path("input") / "arc-prize-2024"
TTT_CKPT_DIR = OUT_DIR / "tmp_training" # local Trainer checkpoint dir (ephemeral container)
# TTT checkpoint/resume: the HF Job container can be restarted mid-run (an undocumented HF
# infra event we saw kill ~5.5h into training), and the local checkpoint dir is ephemeral, so
# each restart otherwise re-runs TTT from step 0. We checkpoint every N steps and upload each
# checkpoint to the experiment's HF artifact repo under "checkpoints/"; on (re)start we pull the
# latest back and resume from it. 0 disables checkpointing. Kept small so a restart loses <= N steps.
CHECKPOINT_EVERY_STEPS = int(os.environ.get("CHECKPOINT_EVERY_STEPS", "100") or "100")
CKPT_REPO_PREFIX = "checkpoints"
DEFAULTS: dict = {
"mode": "without_ttt",
"base_model": "da-fr/Llama-3.2-3B-ARChitects-ReArc-bnb-4bit",
"full_eval": False,
"subset_splits": 40,
"subset_index": 0,
"split_seed": 1234,
"min_prob": 0.1,
"n_guesses": 2,
"infer_repeat": 2,
"infer_seed": 10000,
"ttt_repeat": 48,
}
# The paper's private-test score, recorded alongside our result for context only.
REPORTED_PRIVATE_TEST_SCORE = 0.535
def load_hyperparameters() -> dict:
raw = os.environ.get("HYPERPARAMETERS", "{}")
try:
overrides = json.loads(raw) if raw.strip() else {}
except json.JSONDecodeError as exc:
raise SystemExit(f"HYPERPARAMETERS is not valid JSON: {exc}") from exc
if not isinstance(overrides, dict):
raise SystemExit("HYPERPARAMETERS must be a JSON object")
unknown = set(overrides) - set(DEFAULTS)
if unknown:
raise SystemExit(f"Unknown hyperparameters: {sorted(unknown)}")
return {**DEFAULTS, **overrides}
def build_eval_set(hp: dict):
"""Load the ARC-AGI-1 public evaluation split, optionally reduced to one shard.
Subsetting uses ``split()`` rather than ``change_keys()``: ``change_keys()`` constructs
without ``is_orig``, and ``get_submission()``/``validate_submission()`` both assert it,
so a ``change_keys`` subset would fail at scoring time — after inference has run.
"""
from arc_loader import ArcDataset # noqa: E402 (vendored; needs sys.path above)
from arc_downloader import download_arc_data # noqa: E402
download_arc_data(str(ARC_DATA_PATH))
challenges = ARC_DATA_PATH / "arc-agi_evaluation_challenges.json"
solutions = ARC_DATA_PATH / "arc-agi_evaluation_solutions.json"
eval_set = ArcDataset.load_from_json(str(challenges))
if not hp["full_eval"]:
shards = eval_set.split(n=hp["subset_splits"], split_seed=hp["split_seed"])
eval_set = shards[hp["subset_index"]]
return eval_set.load_solutions(str(solutions))
def _ckpt_repo_dir(name: str) -> str:
"""Path-in-repo for a checkpoint dir (e.g. 'checkpoints/checkpoint-100')."""
return f"{CKPT_REPO_PREFIX}/{name}"
def _upload_checkpoint(local_ckpt: Path) -> None:
"""Upload a Trainer checkpoint dir to the experiment's HF artifact repo.
Best-effort: a failed upload must not kill training (the checkpoint still exists locally
for the rest of this container's life). Uses upload_folder, present in the container's
pinned huggingface_hub 0.25.2.
"""
repo_id = os.environ.get("HF_REPO_ID")
token = os.environ.get("HF_TOKEN")
if not (repo_id and token):
return
try:
from huggingface_hub import HfApi
HfApi(token=token).upload_folder(
folder_path=str(local_ckpt),
path_in_repo=_ckpt_repo_dir(local_ckpt.name),
repo_id=repo_id,
repo_type="model",
)
print(f"[ckpt] uploaded {local_ckpt.name} to {repo_id}", flush=True)
except Exception as exc:
print(f"[ckpt] upload failed ({exc.__class__.__name__}: {exc})", flush=True)
def _download_latest_checkpoint(dest_dir: Path) -> str | None:
"""If the artifact repo holds checkpoints from a prior (restarted) attempt, pull the
latest into dest_dir and return its local path for resume_from_checkpoint; else None.
"""
repo_id = os.environ.get("HF_REPO_ID")
token = os.environ.get("HF_TOKEN")
if not (repo_id and token):
return None
try:
from huggingface_hub import HfApi, snapshot_download
files = HfApi(token=token).list_repo_files(repo_id, repo_type="model")
steps = set()
for f in files:
# checkpoints/checkpoint-<step>/...
parts = f.split("/")
if len(parts) >= 2 and parts[0] == CKPT_REPO_PREFIX and parts[1].startswith("checkpoint-"):
try:
steps.add(int(parts[1].split("-", 1)[1]))
except ValueError:
pass
if not steps:
return None
latest = f"checkpoint-{max(steps)}"
local_root = snapshot_download(
repo_id=repo_id,
repo_type="model",
allow_patterns=f"{CKPT_REPO_PREFIX}/{latest}/*",
token=token,
)
resume_path = Path(local_root) / CKPT_REPO_PREFIX / latest
# Trainer expects the checkpoint dir under its output_dir; mirror it there.
dest_dir.mkdir(parents=True, exist_ok=True)
target = dest_dir / latest
if not target.exists():
import shutil
shutil.copytree(resume_path, target)
print(f"[ckpt] resuming from {latest} (pulled from {repo_id})", flush=True)
return str(target)
except Exception as exc:
print(f"[ckpt] resume-download failed ({exc.__class__.__name__}: {exc})", flush=True)
return None
def run_test_time_training(model, tokenizer, eval_set, fmt_opts, hp):
"""Upstream's TTT block: LoRA fine-tune on the eval tasks' demonstration pairs.
The model never sees the answers — only the demo pairs and the input grids.
"""
from datasets import Dataset
from transformers import TrainerCallback
from unsloth import FastLanguageModel, UnslothTrainer as Trainer
from unsloth import UnslothTrainingArguments as TrainingArguments
from unsloth import is_bfloat16_supported
from model_tools import InputMaskingDataCollator
ckpt_on = CHECKPOINT_EVERY_STEPS > 0
class _UploadCheckpointCallback(TrainerCallback):
"""After the Trainer writes a checkpoint, push it to the artifact repo so it survives
a container restart."""
def on_save(self, args, state, control, **kwargs): # noqa: ARG002
ckpt = TTT_CKPT_DIR / f"checkpoint-{state.global_step}"
if ckpt.exists():
_upload_checkpoint(ckpt)
model = FastLanguageModel.get_peft_model(
model=model,
target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj",
"down_proj", "embed_tokens", "lm_head"],
r=64,
lora_alpha=16,
lora_dropout=0,
bias="none",
use_gradient_checkpointing=True,
random_state=42,
use_rslora=True,
loftq_config=None,
)
train_aug_opts = dict(tp=True, rt=True, perm=True, shfl_ex=True, seed=0)
train_set = eval_set.remove_test_data().repeat(n=hp["ttt_repeat"], seed=0).augment(**train_aug_opts)
FastLanguageModel.for_training(model)
trainer = Trainer(
model=model,
tokenizer=tokenizer,
train_dataset=Dataset.from_list(train_set.as_list(len_name="text", **fmt_opts)),
dataset_text_field="text",
max_seq_length=fmt_opts["max_tokens"],
data_collator=InputMaskingDataCollator(
instruction_template=fmt_opts["query_beg"],
response_template=fmt_opts["reply_beg"],
mlm=False,
tokenizer=tokenizer,
mask_first_n_examples=0,
),
args=TrainingArguments(
per_device_train_batch_size=2,
gradient_accumulation_steps=2,
warmup_ratio=0.25,
num_train_epochs=1,
learning_rate=1e-4,
embedding_learning_rate=1e-5,
fp16=not is_bfloat16_supported(),
bf16=is_bfloat16_supported(),
logging_steps=10,
optim="adamw_8bit",
weight_decay=0.0,
lr_scheduler_type="cosine",
seed=42,
output_dir=str(TTT_CKPT_DIR),
save_strategy="steps" if ckpt_on else "no",
save_steps=CHECKPOINT_EVERY_STEPS if ckpt_on else 500,
save_total_limit=1,
report_to="none",
),
)
if ckpt_on:
trainer.add_callback(_UploadCheckpointCallback())
# If a prior (restarted) attempt left a checkpoint in the artifact repo, resume from it so
# a container restart doesn't re-run TTT from step 0.
resume = _download_latest_checkpoint(TTT_CKPT_DIR) if ckpt_on else None
# Upstream calls unsloth_train(trainer) here, but that symbol is absent from the
# unsloth version the authors themselves pin (2024.9.post4, installed from PyPI by their
# own download notebook) — their run_evaluation script imports a name its pinned unsloth
# never shipped. UnslothTrainer subclasses trl.SFTTrainer and does not override train(),
# so trainer.train() is the equivalent (and only available) training path.
trainer.train(resume_from_checkpoint=resume)
return model
def capture_lockfile() -> Path:
"""Freeze the exact installed environment to out/requirements.lock.
requirements.txt pins the ~15 top-level packages; pip resolves ~60 transitive deps whose
versions are whatever it served that day. This freezes ALL of them, from the container that
actually ran, so the reproduction is repeatable. Also printed to the log (which survives even
if the artifact push is skipped). Written early in main() so it's captured even if a later
step fails.
"""
import subprocess
lock_path = OUT_DIR / "requirements.lock"
try:
frozen = subprocess.run(
[sys.executable, "-m", "pip", "freeze"],
capture_output=True, text=True, check=True,
).stdout
lock_path.write_text(frozen)
print("----- BEGIN requirements.lock (pip freeze) -----", flush=True)
print(frozen, flush=True)
print("----- END requirements.lock -----", flush=True)
except Exception as exc:
print(f"[lockfile] pip freeze failed ({exc.__class__.__name__}: {exc})", flush=True)
return lock_path
def publish_artifacts(payload: dict, submission_path: Path, lock_path: Path) -> None:
"""Push results to the experiment's HF artifact repo, if the harness provided one.
The job container is ephemeral, so anything not pushed is lost when it exits.
"""
repo_id = os.environ.get("HF_REPO_ID")
token = os.environ.get("HF_TOKEN")
if not (repo_id and token):
print("[artifacts] HF_REPO_ID/HF_TOKEN unset; skipping upload", flush=True)
return
try:
from huggingface_hub import HfApi
api = HfApi(token=token)
result_path = OUT_DIR / "result.json"
result_path.write_text(json.dumps(payload, indent=2))
uploads = [result_path, submission_path]
if lock_path.exists():
uploads.append(lock_path)
for path in uploads:
api.upload_file(
path_or_fileobj=str(path),
path_in_repo=path.name,
repo_id=repo_id,
repo_type="model",
)
print(f"[artifacts] uploaded {', '.join(p.name for p in uploads)} to {repo_id}", flush=True)
except Exception as exc: # non-fatal: the score is already in the log
print(f"[artifacts] upload failed ({exc.__class__.__name__}: {exc})", flush=True)
def main() -> None:
hp = load_hyperparameters()
if hp["mode"] not in {"without_ttt", "with_ttt"}:
raise SystemExit(f"mode must be 'without_ttt' or 'with_ttt', got {hp['mode']!r}")
OUT_DIR.mkdir(parents=True, exist_ok=True)
print(f"Experiment: {os.environ.get('EXPERIMENT_ID', '<unset>')}", flush=True)
print(f"Hyperparameters: {json.dumps(hp)}", flush=True)
# GPU-side preflight: the CPU Space-build guard can't import unsloth (it probes for a
# GPU), so unsloth's own API surface is unchecked until runtime. The TTT path imports
# several unsloth symbols lazily inside run_test_time_training(), and a missing one
# otherwise only surfaces after the model has already been downloaded and loaded (this is
# how the unsloth_train import mismatch first burned GPU setup time). Resolve those
# imports here, first thing, so a bad symbol fails in seconds rather than after minutes.
if hp["mode"] == "with_ttt":
import unsloth # GPU-only import; safe here because this is the GPU job
for _sym in ("FastLanguageModel", "UnslothTrainer", "UnslothTrainingArguments",
"is_bfloat16_supported"):
if not hasattr(unsloth, _sym):
raise SystemExit(f"TTT preflight: unsloth is missing '{_sym}'")
print("TTT preflight: unsloth trainer symbols present.", flush=True)
# Freeze the environment before anything heavy runs, so we capture it even if inference fails.
lock_path = capture_lockfile()
eval_set = build_eval_set(hp)
n_tasks = len(eval_set.challenge)
print(f"Evaluating {n_tasks} ARC-AGI-1 public-eval tasks (full_eval={hp['full_eval']})", flush=True)
from diskcache import Cache
from unsloth import FastLanguageModel
from inference_tools import inference_run
from model_tools import load_unsloth_4bit
from selection import EvalTool
model, tokenizer = load_unsloth_4bit(hp["base_model"])
# Format options are upstream's, verbatim — they define the grid serialisation the
# released checkpoints were trained on, so they are not tunable.
fmt_opts = dict(
preprompt="ABCDEFGHJKLMNPQRSTUVWXYZabcdefghjklmnpqrstuvwxyz",
query_beg="I",
reply_beg="\n+/-=O",
reply_end="\n" + tokenizer.eos_token,
lines_sep="\n",
max_tokens=128000,
)
if hp["mode"] == "with_ttt":
print("Running test-time training on the evaluation tasks' demo pairs...", flush=True)
model = run_test_time_training(model, tokenizer, eval_set, fmt_opts, hp)
FastLanguageModel.for_inference(model)
infer_aug_opts = dict(tp="all", rt="all", perm=True, shfl_ex=True, seed=hp["infer_seed"])
infer_dataset = eval_set.repeat(hp["infer_repeat"]).augment(**infer_aug_opts)
# diskcache lives in the ephemeral container: a miss just recomputes, but re-runs get
# no speedup and re-bill GPU minutes. Keep smoke-test subsets small.
cache = Cache(str(OUT_DIR / "inference_cache")).memoize(typed=True, ignore={"model_tok", "guess"})
eval_tool = EvalTool(n_guesses=hp["n_guesses"])
inference_results = inference_run(
model_tok=(model, tokenizer),
fmt_opts=fmt_opts,
dataset=infer_dataset,
min_prob=hp["min_prob"],
aug_score_opts=infer_aug_opts,
callback=eval_tool.process_result,
cache=cache,
)
submission_path = OUT_DIR / "submission.json"
submission = eval_set.get_submission(inference_results)
submission_path.write_text(json.dumps(submission))
score = eval_set.validate_submission(submission)
# Upstream's line, kept so logs stay greppable against the original scripts.
print(f"Score for '{submission_path}':", score, flush=True)
payload = {
"score": score,
"n_tasks": n_tasks,
"accuracy": (score / n_tasks) if n_tasks else 0.0,
"mode": hp["mode"],
"base_model": hp["base_model"],
"full_eval": hp["full_eval"],
"benchmark": "ARC-AGI-1 public evaluation split",
"reported_private_test_score": REPORTED_PRIVATE_TEST_SCORE,
"comparable_to_reported": False,
"note": (
"The reported 53.5% is on Kaggle's 100 private test tasks; this score is on the "
"public evaluation split and is not directly comparable."
),
"hyperparameters": hp,
"experiment_id": os.environ.get("EXPERIMENT_ID"),
}
print("RESULT " + json.dumps(payload), flush=True)
publish_artifacts(payload, submission_path, lock_path)
print("Done.", flush=True)
if __name__ == "__main__":
main()