"""Harness entrypoint for the ARChitects ARC Prize 2024 reproduction (SCRUM-80). HF Jobs runs this as ``python /app/train.py``. It reproduces upstream's ``run_evaluation_Llama-rearc_{without,with}_ttt.py`` but takes its parameters from the ``HYPERPARAMETERS`` env var (JSON) that commons.infra injects, because the upstream scripts have no CLI arguments — model, thresholds, augmentation and the full 400-task eval set are all hardcoded there. The ARChitects' code is vendored unmodified under ``vendor/architects/`` (Apache-2.0, Copyright 2024 Daniel Franzen and Jan Disselhoff). Only this file is ours. Dataset: ARC-AGI-1's 400 *public evaluation* tasks, fetched at run time from github.com/fchollet/ARC-AGI by the vendored ``arc_downloader``. NOTE ON SCORES: the paper's headline 53.5% is on Kaggle's 100 *private* test tasks, which are not public. Scores produced here are on the public evaluation split and are NOT directly comparable to 53.5%. Hyperparameters (all optional; defaults shown): mode "without_ttt" | "with_ttt" inference only, or test-time training first base_model HF model id to load in 4-bit full_eval true -> all 400 eval tasks; false -> one shard (see subset_*) subset_splits number of shards to cut the eval set into (40 -> ~10 tasks/shard) subset_index which shard to run split_seed shard permutation seed min_prob DFS probability threshold (upstream: 0.1) n_guesses submitted candidates per task (upstream: 2) infer_repeat dataset repeats before augmentation (upstream: 2) infer_seed augmentation seed (upstream: 10000) ttt_repeat TTT demo-pair repeats (upstream: 48), only used when mode=with_ttt """ from __future__ import annotations import json import os import sys from pathlib import Path # Vendored upstream modules import each other by bare name (``from arc_loader import ...``), # so their directory must be importable directly. _VENDOR = Path(__file__).resolve().parent / "vendor" / "architects" sys.path.insert(0, str(_VENDOR)) OUT_DIR = Path("/app/out") ARC_DATA_PATH = Path("input") / "arc-prize-2024" TTT_CKPT_DIR = OUT_DIR / "tmp_training" # local Trainer checkpoint dir (ephemeral container) # TTT checkpoint/resume: the HF Job container can be restarted mid-run (an undocumented HF # infra event we saw kill ~5.5h into training), and the local checkpoint dir is ephemeral, so # each restart otherwise re-runs TTT from step 0. We checkpoint every N steps and upload each # checkpoint to the experiment's HF artifact repo under "checkpoints/"; on (re)start we pull the # latest back and resume from it. 0 disables checkpointing. Kept small so a restart loses <= N steps. CHECKPOINT_EVERY_STEPS = int(os.environ.get("CHECKPOINT_EVERY_STEPS", "100") or "100") CKPT_REPO_PREFIX = "checkpoints" DEFAULTS: dict = { "mode": "without_ttt", "base_model": "da-fr/Llama-3.2-3B-ARChitects-ReArc-bnb-4bit", "full_eval": False, "subset_splits": 40, "subset_index": 0, "split_seed": 1234, "min_prob": 0.1, "n_guesses": 2, "infer_repeat": 2, "infer_seed": 10000, "ttt_repeat": 48, } # The paper's private-test score, recorded alongside our result for context only. REPORTED_PRIVATE_TEST_SCORE = 0.535 def load_hyperparameters() -> dict: raw = os.environ.get("HYPERPARAMETERS", "{}") try: overrides = json.loads(raw) if raw.strip() else {} except json.JSONDecodeError as exc: raise SystemExit(f"HYPERPARAMETERS is not valid JSON: {exc}") from exc if not isinstance(overrides, dict): raise SystemExit("HYPERPARAMETERS must be a JSON object") unknown = set(overrides) - set(DEFAULTS) if unknown: raise SystemExit(f"Unknown hyperparameters: {sorted(unknown)}") return {**DEFAULTS, **overrides} def build_eval_set(hp: dict): """Load the ARC-AGI-1 public evaluation split, optionally reduced to one shard. Subsetting uses ``split()`` rather than ``change_keys()``: ``change_keys()`` constructs without ``is_orig``, and ``get_submission()``/``validate_submission()`` both assert it, so a ``change_keys`` subset would fail at scoring time — after inference has run. """ from arc_loader import ArcDataset # noqa: E402 (vendored; needs sys.path above) from arc_downloader import download_arc_data # noqa: E402 download_arc_data(str(ARC_DATA_PATH)) challenges = ARC_DATA_PATH / "arc-agi_evaluation_challenges.json" solutions = ARC_DATA_PATH / "arc-agi_evaluation_solutions.json" eval_set = ArcDataset.load_from_json(str(challenges)) if not hp["full_eval"]: shards = eval_set.split(n=hp["subset_splits"], split_seed=hp["split_seed"]) eval_set = shards[hp["subset_index"]] return eval_set.load_solutions(str(solutions)) def _ckpt_repo_dir(name: str) -> str: """Path-in-repo for a checkpoint dir (e.g. 'checkpoints/checkpoint-100').""" return f"{CKPT_REPO_PREFIX}/{name}" def _upload_checkpoint(local_ckpt: Path) -> None: """Upload a Trainer checkpoint dir to the experiment's HF artifact repo. Best-effort: a failed upload must not kill training (the checkpoint still exists locally for the rest of this container's life). Uses upload_folder, present in the container's pinned huggingface_hub 0.25.2. """ repo_id = os.environ.get("HF_REPO_ID") token = os.environ.get("HF_TOKEN") if not (repo_id and token): return try: from huggingface_hub import HfApi HfApi(token=token).upload_folder( folder_path=str(local_ckpt), path_in_repo=_ckpt_repo_dir(local_ckpt.name), repo_id=repo_id, repo_type="model", ) print(f"[ckpt] uploaded {local_ckpt.name} to {repo_id}", flush=True) except Exception as exc: print(f"[ckpt] upload failed ({exc.__class__.__name__}: {exc})", flush=True) def _download_latest_checkpoint(dest_dir: Path) -> str | None: """If the artifact repo holds checkpoints from a prior (restarted) attempt, pull the latest into dest_dir and return its local path for resume_from_checkpoint; else None. """ repo_id = os.environ.get("HF_REPO_ID") token = os.environ.get("HF_TOKEN") if not (repo_id and token): return None try: from huggingface_hub import HfApi, snapshot_download files = HfApi(token=token).list_repo_files(repo_id, repo_type="model") steps = set() for f in files: # checkpoints/checkpoint-/... parts = f.split("/") if len(parts) >= 2 and parts[0] == CKPT_REPO_PREFIX and parts[1].startswith("checkpoint-"): try: steps.add(int(parts[1].split("-", 1)[1])) except ValueError: pass if not steps: return None latest = f"checkpoint-{max(steps)}" local_root = snapshot_download( repo_id=repo_id, repo_type="model", allow_patterns=f"{CKPT_REPO_PREFIX}/{latest}/*", token=token, ) resume_path = Path(local_root) / CKPT_REPO_PREFIX / latest # Trainer expects the checkpoint dir under its output_dir; mirror it there. dest_dir.mkdir(parents=True, exist_ok=True) target = dest_dir / latest if not target.exists(): import shutil shutil.copytree(resume_path, target) print(f"[ckpt] resuming from {latest} (pulled from {repo_id})", flush=True) return str(target) except Exception as exc: print(f"[ckpt] resume-download failed ({exc.__class__.__name__}: {exc})", flush=True) return None def run_test_time_training(model, tokenizer, eval_set, fmt_opts, hp): """Upstream's TTT block: LoRA fine-tune on the eval tasks' demonstration pairs. The model never sees the answers — only the demo pairs and the input grids. """ from datasets import Dataset from transformers import TrainerCallback from unsloth import FastLanguageModel, UnslothTrainer as Trainer from unsloth import UnslothTrainingArguments as TrainingArguments from unsloth import is_bfloat16_supported from model_tools import InputMaskingDataCollator ckpt_on = CHECKPOINT_EVERY_STEPS > 0 class _UploadCheckpointCallback(TrainerCallback): """After the Trainer writes a checkpoint, push it to the artifact repo so it survives a container restart.""" def on_save(self, args, state, control, **kwargs): # noqa: ARG002 ckpt = TTT_CKPT_DIR / f"checkpoint-{state.global_step}" if ckpt.exists(): _upload_checkpoint(ckpt) model = FastLanguageModel.get_peft_model( model=model, target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", "embed_tokens", "lm_head"], r=64, lora_alpha=16, lora_dropout=0, bias="none", use_gradient_checkpointing=True, random_state=42, use_rslora=True, loftq_config=None, ) train_aug_opts = dict(tp=True, rt=True, perm=True, shfl_ex=True, seed=0) train_set = eval_set.remove_test_data().repeat(n=hp["ttt_repeat"], seed=0).augment(**train_aug_opts) FastLanguageModel.for_training(model) trainer = Trainer( model=model, tokenizer=tokenizer, train_dataset=Dataset.from_list(train_set.as_list(len_name="text", **fmt_opts)), dataset_text_field="text", max_seq_length=fmt_opts["max_tokens"], data_collator=InputMaskingDataCollator( instruction_template=fmt_opts["query_beg"], response_template=fmt_opts["reply_beg"], mlm=False, tokenizer=tokenizer, mask_first_n_examples=0, ), args=TrainingArguments( per_device_train_batch_size=2, gradient_accumulation_steps=2, warmup_ratio=0.25, num_train_epochs=1, learning_rate=1e-4, embedding_learning_rate=1e-5, fp16=not is_bfloat16_supported(), bf16=is_bfloat16_supported(), logging_steps=10, optim="adamw_8bit", weight_decay=0.0, lr_scheduler_type="cosine", seed=42, output_dir=str(TTT_CKPT_DIR), save_strategy="steps" if ckpt_on else "no", save_steps=CHECKPOINT_EVERY_STEPS if ckpt_on else 500, save_total_limit=1, report_to="none", ), ) if ckpt_on: trainer.add_callback(_UploadCheckpointCallback()) # If a prior (restarted) attempt left a checkpoint in the artifact repo, resume from it so # a container restart doesn't re-run TTT from step 0. resume = _download_latest_checkpoint(TTT_CKPT_DIR) if ckpt_on else None # Upstream calls unsloth_train(trainer) here, but that symbol is absent from the # unsloth version the authors themselves pin (2024.9.post4, installed from PyPI by their # own download notebook) — their run_evaluation script imports a name its pinned unsloth # never shipped. UnslothTrainer subclasses trl.SFTTrainer and does not override train(), # so trainer.train() is the equivalent (and only available) training path. trainer.train(resume_from_checkpoint=resume) return model def capture_lockfile() -> Path: """Freeze the exact installed environment to out/requirements.lock. requirements.txt pins the ~15 top-level packages; pip resolves ~60 transitive deps whose versions are whatever it served that day. This freezes ALL of them, from the container that actually ran, so the reproduction is repeatable. Also printed to the log (which survives even if the artifact push is skipped). Written early in main() so it's captured even if a later step fails. """ import subprocess lock_path = OUT_DIR / "requirements.lock" try: frozen = subprocess.run( [sys.executable, "-m", "pip", "freeze"], capture_output=True, text=True, check=True, ).stdout lock_path.write_text(frozen) print("----- BEGIN requirements.lock (pip freeze) -----", flush=True) print(frozen, flush=True) print("----- END requirements.lock -----", flush=True) except Exception as exc: print(f"[lockfile] pip freeze failed ({exc.__class__.__name__}: {exc})", flush=True) return lock_path def publish_artifacts(payload: dict, submission_path: Path, lock_path: Path) -> None: """Push results to the experiment's HF artifact repo, if the harness provided one. The job container is ephemeral, so anything not pushed is lost when it exits. """ repo_id = os.environ.get("HF_REPO_ID") token = os.environ.get("HF_TOKEN") if not (repo_id and token): print("[artifacts] HF_REPO_ID/HF_TOKEN unset; skipping upload", flush=True) return try: from huggingface_hub import HfApi api = HfApi(token=token) result_path = OUT_DIR / "result.json" result_path.write_text(json.dumps(payload, indent=2)) uploads = [result_path, submission_path] if lock_path.exists(): uploads.append(lock_path) for path in uploads: api.upload_file( path_or_fileobj=str(path), path_in_repo=path.name, repo_id=repo_id, repo_type="model", ) print(f"[artifacts] uploaded {', '.join(p.name for p in uploads)} to {repo_id}", flush=True) except Exception as exc: # non-fatal: the score is already in the log print(f"[artifacts] upload failed ({exc.__class__.__name__}: {exc})", flush=True) def main() -> None: hp = load_hyperparameters() if hp["mode"] not in {"without_ttt", "with_ttt"}: raise SystemExit(f"mode must be 'without_ttt' or 'with_ttt', got {hp['mode']!r}") OUT_DIR.mkdir(parents=True, exist_ok=True) print(f"Experiment: {os.environ.get('EXPERIMENT_ID', '')}", flush=True) print(f"Hyperparameters: {json.dumps(hp)}", flush=True) # GPU-side preflight: the CPU Space-build guard can't import unsloth (it probes for a # GPU), so unsloth's own API surface is unchecked until runtime. The TTT path imports # several unsloth symbols lazily inside run_test_time_training(), and a missing one # otherwise only surfaces after the model has already been downloaded and loaded (this is # how the unsloth_train import mismatch first burned GPU setup time). Resolve those # imports here, first thing, so a bad symbol fails in seconds rather than after minutes. if hp["mode"] == "with_ttt": import unsloth # GPU-only import; safe here because this is the GPU job for _sym in ("FastLanguageModel", "UnslothTrainer", "UnslothTrainingArguments", "is_bfloat16_supported"): if not hasattr(unsloth, _sym): raise SystemExit(f"TTT preflight: unsloth is missing '{_sym}'") print("TTT preflight: unsloth trainer symbols present.", flush=True) # Freeze the environment before anything heavy runs, so we capture it even if inference fails. lock_path = capture_lockfile() eval_set = build_eval_set(hp) n_tasks = len(eval_set.challenge) print(f"Evaluating {n_tasks} ARC-AGI-1 public-eval tasks (full_eval={hp['full_eval']})", flush=True) from diskcache import Cache from unsloth import FastLanguageModel from inference_tools import inference_run from model_tools import load_unsloth_4bit from selection import EvalTool model, tokenizer = load_unsloth_4bit(hp["base_model"]) # Format options are upstream's, verbatim — they define the grid serialisation the # released checkpoints were trained on, so they are not tunable. fmt_opts = dict( preprompt="ABCDEFGHJKLMNPQRSTUVWXYZabcdefghjklmnpqrstuvwxyz", query_beg="I", reply_beg="\n+/-=O", reply_end="\n" + tokenizer.eos_token, lines_sep="\n", max_tokens=128000, ) if hp["mode"] == "with_ttt": print("Running test-time training on the evaluation tasks' demo pairs...", flush=True) model = run_test_time_training(model, tokenizer, eval_set, fmt_opts, hp) FastLanguageModel.for_inference(model) infer_aug_opts = dict(tp="all", rt="all", perm=True, shfl_ex=True, seed=hp["infer_seed"]) infer_dataset = eval_set.repeat(hp["infer_repeat"]).augment(**infer_aug_opts) # diskcache lives in the ephemeral container: a miss just recomputes, but re-runs get # no speedup and re-bill GPU minutes. Keep smoke-test subsets small. cache = Cache(str(OUT_DIR / "inference_cache")).memoize(typed=True, ignore={"model_tok", "guess"}) eval_tool = EvalTool(n_guesses=hp["n_guesses"]) inference_results = inference_run( model_tok=(model, tokenizer), fmt_opts=fmt_opts, dataset=infer_dataset, min_prob=hp["min_prob"], aug_score_opts=infer_aug_opts, callback=eval_tool.process_result, cache=cache, ) submission_path = OUT_DIR / "submission.json" submission = eval_set.get_submission(inference_results) submission_path.write_text(json.dumps(submission)) score = eval_set.validate_submission(submission) # Upstream's line, kept so logs stay greppable against the original scripts. print(f"Score for '{submission_path}':", score, flush=True) payload = { "score": score, "n_tasks": n_tasks, "accuracy": (score / n_tasks) if n_tasks else 0.0, "mode": hp["mode"], "base_model": hp["base_model"], "full_eval": hp["full_eval"], "benchmark": "ARC-AGI-1 public evaluation split", "reported_private_test_score": REPORTED_PRIVATE_TEST_SCORE, "comparable_to_reported": False, "note": ( "The reported 53.5% is on Kaggle's 100 private test tasks; this score is on the " "public evaluation split and is not directly comparable." ), "hyperparameters": hp, "experiment_id": os.environ.get("EXPERIMENT_ID"), } print("RESULT " + json.dumps(payload), flush=True) publish_artifacts(payload, submission_path, lock_path) print("Done.", flush=True) if __name__ == "__main__": main()