Download train.py from aidevices/arc1-first-solution-image: direct link, hf CLI and curl.
- Browser
- Download file 18.5 kB
-
https://huggingface.co/spaces/aidevices/arc1-first-solution-image/resolve/main/train.py
- Command line
-
hf download hf://spaces/aidevices/arc1-first-solution-image/train.py
-
curl -L -o train.py https://huggingface.co/spaces/aidevices/arc1-first-solution-image/resolve/main/train.py
18.5 kB
| """Harness entrypoint for the ARChitects ARC Prize 2024 reproduction (SCRUM-80). | |
| HF Jobs runs this as ``python /app/train.py``. It reproduces upstream's | |
| ``run_evaluation_Llama-rearc_{without,with}_ttt.py`` but takes its parameters from the | |
| ``HYPERPARAMETERS`` env var (JSON) that commons.infra injects, because the upstream scripts | |
| have no CLI arguments — model, thresholds, augmentation and the full 400-task eval set are | |
| all hardcoded there. | |
| The ARChitects' code is vendored unmodified under ``vendor/architects/`` (Apache-2.0, | |
| Copyright 2024 Daniel Franzen and Jan Disselhoff). Only this file is ours. | |
| Dataset: ARC-AGI-1's 400 *public evaluation* tasks, fetched at run time from | |
| github.com/fchollet/ARC-AGI by the vendored ``arc_downloader``. | |
| NOTE ON SCORES: the paper's headline 53.5% is on Kaggle's 100 *private* test tasks, | |
| which are not public. Scores produced here are on the public evaluation split and are | |
| NOT directly comparable to 53.5%. | |
| Hyperparameters (all optional; defaults shown): | |
| mode "without_ttt" | "with_ttt" inference only, or test-time training first | |
| base_model HF model id to load in 4-bit | |
| full_eval true -> all 400 eval tasks; false -> one shard (see subset_*) | |
| subset_splits number of shards to cut the eval set into (40 -> ~10 tasks/shard) | |
| subset_index which shard to run | |
| split_seed shard permutation seed | |
| min_prob DFS probability threshold (upstream: 0.1) | |
| n_guesses submitted candidates per task (upstream: 2) | |
| infer_repeat dataset repeats before augmentation (upstream: 2) | |
| infer_seed augmentation seed (upstream: 10000) | |
| ttt_repeat TTT demo-pair repeats (upstream: 48), only used when mode=with_ttt | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import sys | |
| from pathlib import Path | |
| # Vendored upstream modules import each other by bare name (``from arc_loader import ...``), | |
| # so their directory must be importable directly. | |
| _VENDOR = Path(__file__).resolve().parent / "vendor" / "architects" | |
| sys.path.insert(0, str(_VENDOR)) | |
| OUT_DIR = Path("/app/out") | |
| ARC_DATA_PATH = Path("input") / "arc-prize-2024" | |
| TTT_CKPT_DIR = OUT_DIR / "tmp_training" # local Trainer checkpoint dir (ephemeral container) | |
| # TTT checkpoint/resume: the HF Job container can be restarted mid-run (an undocumented HF | |
| # infra event we saw kill ~5.5h into training), and the local checkpoint dir is ephemeral, so | |
| # each restart otherwise re-runs TTT from step 0. We checkpoint every N steps and upload each | |
| # checkpoint to the experiment's HF artifact repo under "checkpoints/"; on (re)start we pull the | |
| # latest back and resume from it. 0 disables checkpointing. Kept small so a restart loses <= N steps. | |
| CHECKPOINT_EVERY_STEPS = int(os.environ.get("CHECKPOINT_EVERY_STEPS", "100") or "100") | |
| CKPT_REPO_PREFIX = "checkpoints" | |
| DEFAULTS: dict = { | |
| "mode": "without_ttt", | |
| "base_model": "da-fr/Llama-3.2-3B-ARChitects-ReArc-bnb-4bit", | |
| "full_eval": False, | |
| "subset_splits": 40, | |
| "subset_index": 0, | |
| "split_seed": 1234, | |
| "min_prob": 0.1, | |
| "n_guesses": 2, | |
| "infer_repeat": 2, | |
| "infer_seed": 10000, | |
| "ttt_repeat": 48, | |
| } | |
| # The paper's private-test score, recorded alongside our result for context only. | |
| REPORTED_PRIVATE_TEST_SCORE = 0.535 | |
| def load_hyperparameters() -> dict: | |
| raw = os.environ.get("HYPERPARAMETERS", "{}") | |
| try: | |
| overrides = json.loads(raw) if raw.strip() else {} | |
| except json.JSONDecodeError as exc: | |
| raise SystemExit(f"HYPERPARAMETERS is not valid JSON: {exc}") from exc | |
| if not isinstance(overrides, dict): | |
| raise SystemExit("HYPERPARAMETERS must be a JSON object") | |
| unknown = set(overrides) - set(DEFAULTS) | |
| if unknown: | |
| raise SystemExit(f"Unknown hyperparameters: {sorted(unknown)}") | |
| return {**DEFAULTS, **overrides} | |
| def build_eval_set(hp: dict): | |
| """Load the ARC-AGI-1 public evaluation split, optionally reduced to one shard. | |
| Subsetting uses ``split()`` rather than ``change_keys()``: ``change_keys()`` constructs | |
| without ``is_orig``, and ``get_submission()``/``validate_submission()`` both assert it, | |
| so a ``change_keys`` subset would fail at scoring time — after inference has run. | |
| """ | |
| from arc_loader import ArcDataset # noqa: E402 (vendored; needs sys.path above) | |
| from arc_downloader import download_arc_data # noqa: E402 | |
| download_arc_data(str(ARC_DATA_PATH)) | |
| challenges = ARC_DATA_PATH / "arc-agi_evaluation_challenges.json" | |
| solutions = ARC_DATA_PATH / "arc-agi_evaluation_solutions.json" | |
| eval_set = ArcDataset.load_from_json(str(challenges)) | |
| if not hp["full_eval"]: | |
| shards = eval_set.split(n=hp["subset_splits"], split_seed=hp["split_seed"]) | |
| eval_set = shards[hp["subset_index"]] | |
| return eval_set.load_solutions(str(solutions)) | |
| def _ckpt_repo_dir(name: str) -> str: | |
| """Path-in-repo for a checkpoint dir (e.g. 'checkpoints/checkpoint-100').""" | |
| return f"{CKPT_REPO_PREFIX}/{name}" | |
| def _upload_checkpoint(local_ckpt: Path) -> None: | |
| """Upload a Trainer checkpoint dir to the experiment's HF artifact repo. | |
| Best-effort: a failed upload must not kill training (the checkpoint still exists locally | |
| for the rest of this container's life). Uses upload_folder, present in the container's | |
| pinned huggingface_hub 0.25.2. | |
| """ | |
| repo_id = os.environ.get("HF_REPO_ID") | |
| token = os.environ.get("HF_TOKEN") | |
| if not (repo_id and token): | |
| return | |
| try: | |
| from huggingface_hub import HfApi | |
| HfApi(token=token).upload_folder( | |
| folder_path=str(local_ckpt), | |
| path_in_repo=_ckpt_repo_dir(local_ckpt.name), | |
| repo_id=repo_id, | |
| repo_type="model", | |
| ) | |
| print(f"[ckpt] uploaded {local_ckpt.name} to {repo_id}", flush=True) | |
| except Exception as exc: | |
| print(f"[ckpt] upload failed ({exc.__class__.__name__}: {exc})", flush=True) | |
| def _download_latest_checkpoint(dest_dir: Path) -> str | None: | |
| """If the artifact repo holds checkpoints from a prior (restarted) attempt, pull the | |
| latest into dest_dir and return its local path for resume_from_checkpoint; else None. | |
| """ | |
| repo_id = os.environ.get("HF_REPO_ID") | |
| token = os.environ.get("HF_TOKEN") | |
| if not (repo_id and token): | |
| return None | |
| try: | |
| from huggingface_hub import HfApi, snapshot_download | |
| files = HfApi(token=token).list_repo_files(repo_id, repo_type="model") | |
| steps = set() | |
| for f in files: | |
| # checkpoints/checkpoint-<step>/... | |
| parts = f.split("/") | |
| if len(parts) >= 2 and parts[0] == CKPT_REPO_PREFIX and parts[1].startswith("checkpoint-"): | |
| try: | |
| steps.add(int(parts[1].split("-", 1)[1])) | |
| except ValueError: | |
| pass | |
| if not steps: | |
| return None | |
| latest = f"checkpoint-{max(steps)}" | |
| local_root = snapshot_download( | |
| repo_id=repo_id, | |
| repo_type="model", | |
| allow_patterns=f"{CKPT_REPO_PREFIX}/{latest}/*", | |
| token=token, | |
| ) | |
| resume_path = Path(local_root) / CKPT_REPO_PREFIX / latest | |
| # Trainer expects the checkpoint dir under its output_dir; mirror it there. | |
| dest_dir.mkdir(parents=True, exist_ok=True) | |
| target = dest_dir / latest | |
| if not target.exists(): | |
| import shutil | |
| shutil.copytree(resume_path, target) | |
| print(f"[ckpt] resuming from {latest} (pulled from {repo_id})", flush=True) | |
| return str(target) | |
| except Exception as exc: | |
| print(f"[ckpt] resume-download failed ({exc.__class__.__name__}: {exc})", flush=True) | |
| return None | |
| def run_test_time_training(model, tokenizer, eval_set, fmt_opts, hp): | |
| """Upstream's TTT block: LoRA fine-tune on the eval tasks' demonstration pairs. | |
| The model never sees the answers — only the demo pairs and the input grids. | |
| """ | |
| from datasets import Dataset | |
| from transformers import TrainerCallback | |
| from unsloth import FastLanguageModel, UnslothTrainer as Trainer | |
| from unsloth import UnslothTrainingArguments as TrainingArguments | |
| from unsloth import is_bfloat16_supported | |
| from model_tools import InputMaskingDataCollator | |
| ckpt_on = CHECKPOINT_EVERY_STEPS > 0 | |
| class _UploadCheckpointCallback(TrainerCallback): | |
| """After the Trainer writes a checkpoint, push it to the artifact repo so it survives | |
| a container restart.""" | |
| def on_save(self, args, state, control, **kwargs): # noqa: ARG002 | |
| ckpt = TTT_CKPT_DIR / f"checkpoint-{state.global_step}" | |
| if ckpt.exists(): | |
| _upload_checkpoint(ckpt) | |
| model = FastLanguageModel.get_peft_model( | |
| model=model, | |
| target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", | |
| "down_proj", "embed_tokens", "lm_head"], | |
| r=64, | |
| lora_alpha=16, | |
| lora_dropout=0, | |
| bias="none", | |
| use_gradient_checkpointing=True, | |
| random_state=42, | |
| use_rslora=True, | |
| loftq_config=None, | |
| ) | |
| train_aug_opts = dict(tp=True, rt=True, perm=True, shfl_ex=True, seed=0) | |
| train_set = eval_set.remove_test_data().repeat(n=hp["ttt_repeat"], seed=0).augment(**train_aug_opts) | |
| FastLanguageModel.for_training(model) | |
| trainer = Trainer( | |
| model=model, | |
| tokenizer=tokenizer, | |
| train_dataset=Dataset.from_list(train_set.as_list(len_name="text", **fmt_opts)), | |
| dataset_text_field="text", | |
| max_seq_length=fmt_opts["max_tokens"], | |
| data_collator=InputMaskingDataCollator( | |
| instruction_template=fmt_opts["query_beg"], | |
| response_template=fmt_opts["reply_beg"], | |
| mlm=False, | |
| tokenizer=tokenizer, | |
| mask_first_n_examples=0, | |
| ), | |
| args=TrainingArguments( | |
| per_device_train_batch_size=2, | |
| gradient_accumulation_steps=2, | |
| warmup_ratio=0.25, | |
| num_train_epochs=1, | |
| learning_rate=1e-4, | |
| embedding_learning_rate=1e-5, | |
| fp16=not is_bfloat16_supported(), | |
| bf16=is_bfloat16_supported(), | |
| logging_steps=10, | |
| optim="adamw_8bit", | |
| weight_decay=0.0, | |
| lr_scheduler_type="cosine", | |
| seed=42, | |
| output_dir=str(TTT_CKPT_DIR), | |
| save_strategy="steps" if ckpt_on else "no", | |
| save_steps=CHECKPOINT_EVERY_STEPS if ckpt_on else 500, | |
| save_total_limit=1, | |
| report_to="none", | |
| ), | |
| ) | |
| if ckpt_on: | |
| trainer.add_callback(_UploadCheckpointCallback()) | |
| # If a prior (restarted) attempt left a checkpoint in the artifact repo, resume from it so | |
| # a container restart doesn't re-run TTT from step 0. | |
| resume = _download_latest_checkpoint(TTT_CKPT_DIR) if ckpt_on else None | |
| # Upstream calls unsloth_train(trainer) here, but that symbol is absent from the | |
| # unsloth version the authors themselves pin (2024.9.post4, installed from PyPI by their | |
| # own download notebook) — their run_evaluation script imports a name its pinned unsloth | |
| # never shipped. UnslothTrainer subclasses trl.SFTTrainer and does not override train(), | |
| # so trainer.train() is the equivalent (and only available) training path. | |
| trainer.train(resume_from_checkpoint=resume) | |
| return model | |
| def capture_lockfile() -> Path: | |
| """Freeze the exact installed environment to out/requirements.lock. | |
| requirements.txt pins the ~15 top-level packages; pip resolves ~60 transitive deps whose | |
| versions are whatever it served that day. This freezes ALL of them, from the container that | |
| actually ran, so the reproduction is repeatable. Also printed to the log (which survives even | |
| if the artifact push is skipped). Written early in main() so it's captured even if a later | |
| step fails. | |
| """ | |
| import subprocess | |
| lock_path = OUT_DIR / "requirements.lock" | |
| try: | |
| frozen = subprocess.run( | |
| [sys.executable, "-m", "pip", "freeze"], | |
| capture_output=True, text=True, check=True, | |
| ).stdout | |
| lock_path.write_text(frozen) | |
| print("----- BEGIN requirements.lock (pip freeze) -----", flush=True) | |
| print(frozen, flush=True) | |
| print("----- END requirements.lock -----", flush=True) | |
| except Exception as exc: | |
| print(f"[lockfile] pip freeze failed ({exc.__class__.__name__}: {exc})", flush=True) | |
| return lock_path | |
| def publish_artifacts(payload: dict, submission_path: Path, lock_path: Path) -> None: | |
| """Push results to the experiment's HF artifact repo, if the harness provided one. | |
| The job container is ephemeral, so anything not pushed is lost when it exits. | |
| """ | |
| repo_id = os.environ.get("HF_REPO_ID") | |
| token = os.environ.get("HF_TOKEN") | |
| if not (repo_id and token): | |
| print("[artifacts] HF_REPO_ID/HF_TOKEN unset; skipping upload", flush=True) | |
| return | |
| try: | |
| from huggingface_hub import HfApi | |
| api = HfApi(token=token) | |
| result_path = OUT_DIR / "result.json" | |
| result_path.write_text(json.dumps(payload, indent=2)) | |
| uploads = [result_path, submission_path] | |
| if lock_path.exists(): | |
| uploads.append(lock_path) | |
| for path in uploads: | |
| api.upload_file( | |
| path_or_fileobj=str(path), | |
| path_in_repo=path.name, | |
| repo_id=repo_id, | |
| repo_type="model", | |
| ) | |
| print(f"[artifacts] uploaded {', '.join(p.name for p in uploads)} to {repo_id}", flush=True) | |
| except Exception as exc: # non-fatal: the score is already in the log | |
| print(f"[artifacts] upload failed ({exc.__class__.__name__}: {exc})", flush=True) | |
| def main() -> None: | |
| hp = load_hyperparameters() | |
| if hp["mode"] not in {"without_ttt", "with_ttt"}: | |
| raise SystemExit(f"mode must be 'without_ttt' or 'with_ttt', got {hp['mode']!r}") | |
| OUT_DIR.mkdir(parents=True, exist_ok=True) | |
| print(f"Experiment: {os.environ.get('EXPERIMENT_ID', '<unset>')}", flush=True) | |
| print(f"Hyperparameters: {json.dumps(hp)}", flush=True) | |
| # GPU-side preflight: the CPU Space-build guard can't import unsloth (it probes for a | |
| # GPU), so unsloth's own API surface is unchecked until runtime. The TTT path imports | |
| # several unsloth symbols lazily inside run_test_time_training(), and a missing one | |
| # otherwise only surfaces after the model has already been downloaded and loaded (this is | |
| # how the unsloth_train import mismatch first burned GPU setup time). Resolve those | |
| # imports here, first thing, so a bad symbol fails in seconds rather than after minutes. | |
| if hp["mode"] == "with_ttt": | |
| import unsloth # GPU-only import; safe here because this is the GPU job | |
| for _sym in ("FastLanguageModel", "UnslothTrainer", "UnslothTrainingArguments", | |
| "is_bfloat16_supported"): | |
| if not hasattr(unsloth, _sym): | |
| raise SystemExit(f"TTT preflight: unsloth is missing '{_sym}'") | |
| print("TTT preflight: unsloth trainer symbols present.", flush=True) | |
| # Freeze the environment before anything heavy runs, so we capture it even if inference fails. | |
| lock_path = capture_lockfile() | |
| eval_set = build_eval_set(hp) | |
| n_tasks = len(eval_set.challenge) | |
| print(f"Evaluating {n_tasks} ARC-AGI-1 public-eval tasks (full_eval={hp['full_eval']})", flush=True) | |
| from diskcache import Cache | |
| from unsloth import FastLanguageModel | |
| from inference_tools import inference_run | |
| from model_tools import load_unsloth_4bit | |
| from selection import EvalTool | |
| model, tokenizer = load_unsloth_4bit(hp["base_model"]) | |
| # Format options are upstream's, verbatim — they define the grid serialisation the | |
| # released checkpoints were trained on, so they are not tunable. | |
| fmt_opts = dict( | |
| preprompt="ABCDEFGHJKLMNPQRSTUVWXYZabcdefghjklmnpqrstuvwxyz", | |
| query_beg="I", | |
| reply_beg="\n+/-=O", | |
| reply_end="\n" + tokenizer.eos_token, | |
| lines_sep="\n", | |
| max_tokens=128000, | |
| ) | |
| if hp["mode"] == "with_ttt": | |
| print("Running test-time training on the evaluation tasks' demo pairs...", flush=True) | |
| model = run_test_time_training(model, tokenizer, eval_set, fmt_opts, hp) | |
| FastLanguageModel.for_inference(model) | |
| infer_aug_opts = dict(tp="all", rt="all", perm=True, shfl_ex=True, seed=hp["infer_seed"]) | |
| infer_dataset = eval_set.repeat(hp["infer_repeat"]).augment(**infer_aug_opts) | |
| # diskcache lives in the ephemeral container: a miss just recomputes, but re-runs get | |
| # no speedup and re-bill GPU minutes. Keep smoke-test subsets small. | |
| cache = Cache(str(OUT_DIR / "inference_cache")).memoize(typed=True, ignore={"model_tok", "guess"}) | |
| eval_tool = EvalTool(n_guesses=hp["n_guesses"]) | |
| inference_results = inference_run( | |
| model_tok=(model, tokenizer), | |
| fmt_opts=fmt_opts, | |
| dataset=infer_dataset, | |
| min_prob=hp["min_prob"], | |
| aug_score_opts=infer_aug_opts, | |
| callback=eval_tool.process_result, | |
| cache=cache, | |
| ) | |
| submission_path = OUT_DIR / "submission.json" | |
| submission = eval_set.get_submission(inference_results) | |
| submission_path.write_text(json.dumps(submission)) | |
| score = eval_set.validate_submission(submission) | |
| # Upstream's line, kept so logs stay greppable against the original scripts. | |
| print(f"Score for '{submission_path}':", score, flush=True) | |
| payload = { | |
| "score": score, | |
| "n_tasks": n_tasks, | |
| "accuracy": (score / n_tasks) if n_tasks else 0.0, | |
| "mode": hp["mode"], | |
| "base_model": hp["base_model"], | |
| "full_eval": hp["full_eval"], | |
| "benchmark": "ARC-AGI-1 public evaluation split", | |
| "reported_private_test_score": REPORTED_PRIVATE_TEST_SCORE, | |
| "comparable_to_reported": False, | |
| "note": ( | |
| "The reported 53.5% is on Kaggle's 100 private test tasks; this score is on the " | |
| "public evaluation split and is not directly comparable." | |
| ), | |
| "hyperparameters": hp, | |
| "experiment_id": os.environ.get("EXPERIMENT_ID"), | |
| } | |
| print("RESULT " + json.dumps(payload), flush=True) | |
| publish_artifacts(payload, submission_path, lock_path) | |
| print("Done.", flush=True) | |
| if __name__ == "__main__": | |
| main() | |