"""Run explicit B/C/D/E experiment configurations without inventing results. JSON input is a list of ExperimentConfig records or {"experiments": [...]}. Data/checkpoint paths are mandatory to execute; E checks excluded_families in run.json and D checks configuration_shift in the supplied dataset manifest. """ from __future__ import annotations import argparse import json from pathlib import Path from .benchmark import benchmark from .benchmark import parser as benchmark_parser from .experiments import ExperimentConfig, run_experiments def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--config", required=True) parser.add_argument("--output", required=True) parser.add_argument("--samples", type=int, default=512) parser.add_argument("--device", choices=("auto", "cpu", "mps"), default="auto") args = parser.parse_args(argv) content = json.loads(Path(args.config).read_text()) records = content["experiments"] if isinstance(content, dict) else content configs = [ExperimentConfig(**record) for record in records] output = Path(args.output) output.mkdir(parents=True, exist_ok=True) def evaluate(config): if Path(config.name).name != config.name or config.name in (".", ".."): raise ValueError("Experiment names must be a single safe directory name") parsed = benchmark_parser().parse_args( [ "--data", config.data, "--checkpoint", config.checkpoint, "--output", str(output / config.name), "--samples", str(args.samples), "--device", args.device, ] ) parsed.experiment = config result = benchmark(parsed) return {"benchmark": f"{config.name}/benchmark.json", "samples": result["samples"]} run_experiments(configs, evaluate, output / "experiments.json") if __name__ == "__main__": main()