"""Deterministically select fresh, test-backed tasks for Study 4. This outcome-blind selector reads repository history and test outcomes only. It excludes every commit already present in the task catalog, validates that public tests pass at the parent, hidden tests expose the bug, and the gold production patch repairs it, then retains every candidate decision. """ from __future__ import annotations import argparse from dataclasses import asdict import json from pathlib import Path from build_study2_tasks import ( candidate_commits, changed_symbols, manifest_text, precheck, run_git, validate_candidate, load_repository, ) from agent_harness.specs import load_tasks REPOSITORIES = ("R001", "R002", "R003") def select_fresh(root: Path, repository_id: str, count: int, write: bool) -> dict: rule = load_repository(root, repository_id) catalog = load_tasks(root) excluded_commits = {task.gold_commit for task in catalog.values()} selected: list[str] = [] audits = [] manifest_dir = root / "tasks" / "manifests" patch_dir = root / "tasks" / "patches" validation_dir = root / "tasks" / "validation" / "study4" selection_dir = root / "tasks" / "selection" / "study4" if write: validation_dir.mkdir(parents=True, exist_ok=True) selection_dir.mkdir(parents=True, exist_ok=True) for commit in candidate_commits(rule): if len(selected) >= count: break audit = precheck(rule, commit, excluded_commits) audits.append(audit) if audit.status != "eligible": continue assert audit.parent is not None source_patch = str( run_git(rule.path, "diff", "--binary", audit.parent, commit, "--", *audit.source_paths) ) test_patch = str( run_git(rule.path, "diff", "--binary", audit.parent, commit, "--", *audit.test_paths) ) validation = validate_candidate(rule, audit, source_patch, test_patch) audit.validation = validation audit.elapsed_seconds += float(validation["elapsed_seconds"]) if not validation["valid"]: audit.status = "rejected" audit.reason = str(validation["reason"]) print(f"REJECT {repository_id} {commit[:12]} {audit.reason}", flush=True) continue task_id = f"TASK_S4_{repository_id}_{len(selected) + 1:03d}" source_name = f"{task_id}_source.patch" test_name = f"{task_id}_tests.patch" symbols = changed_symbols(rule, audit) audit.task_id = task_id audit.status = "selected" audit.reason = None selected.append(task_id) excluded_commits.add(commit) if write: text = manifest_text( rule, audit, task_id, source_name, test_name, symbols ).replace( "deterministic Study 2 held-out validation", "deterministic Study 4 fresh-task validation", ) (manifest_dir / f"{task_id}.toml").write_text(text, encoding="utf-8") (patch_dir / source_name).write_text(source_patch, encoding="utf-8") (patch_dir / test_name).write_text(test_patch, encoding="utf-8") (validation_dir / f"{task_id}.json").write_text( json.dumps( { "schema_version": 1, "study": "Study 4 fresh-task retrieval replication", "task_id": task_id, "repository_id": repository_id, "commit": commit, "parent": audit.parent, "valid_end_to_end": True, "validation": validation, }, indent=2, sort_keys=True, ) + "\n", encoding="utf-8", ) print( f"SELECT {task_id} {commit[:12]} source={len(audit.source_paths)} " f"tests={len(audit.test_paths)} lines={audit.source_changed_lines}", flush=True, ) report = { "schema_version": 1, "study": "Study 4 fresh-task retrieval replication", "repository": asdict(rule) | {"path": str(rule.path)}, "selection_policy": { "fresh_count": count, "excluded_prior_task_commits": len({task.gold_commit for task in catalog.values()}), "order": "reverse chronological first-parent history since 2023-01-01", "outcome_blind": True, }, "selected_task_ids": selected, "selected_count": len(selected), "complete": len(selected) == count, "audits": [asdict(item) for item in audits], } if write: (selection_dir / f"{repository_id}_selection.json").write_text( json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" ) if len(selected) != count: raise RuntimeError(f"{repository_id} yielded {len(selected)}/{count} fresh valid tasks") return report def freeze_split(root: Path, count: int) -> tuple[str, ...]: catalog = load_tasks(root) expected = [ f"TASK_S4_{repository_id}_{number:03d}" for repository_id in REPOSITORIES for number in range(1, count + 1) ] missing = [task_id for task_id in expected if task_id not in catalog] if missing: raise RuntimeError(f"cannot freeze Study 4 split; missing {missing}") path = root / "tasks" / "splits" / "study4_fresh.txt" path.write_text( "# Fresh Study 4 split; frozen before any E10 LLM inference.\n" + "\n".join(expected) + "\n", encoding="utf-8", ) return tuple(expected) def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) parser.add_argument("--repository-id", choices=REPOSITORIES) parser.add_argument("--count", type=int, default=10) parser.add_argument("--write", action="store_true") parser.add_argument("--freeze-split", action="store_true") args = parser.parse_args() root = args.root.resolve() if args.freeze_split: split = freeze_split(root, args.count) print(json.dumps({"split_count": len(split), "task_ids": split}, indent=2)) return if not args.repository_id: parser.error("--repository-id is required unless --freeze-split is used") report = select_fresh(root, args.repository_id, args.count, args.write) print(json.dumps({"repository_id": args.repository_id, "selected": report["selected_count"]})) if __name__ == "__main__": main()