File size: 2,048 Bytes
1baf215 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | """Run explicit B/C/D/E experiment configurations without inventing results.
JSON input is a list of ExperimentConfig records or {"experiments": [...]}.
Data/checkpoint paths are mandatory to execute; E checks excluded_families in
run.json and D checks configuration_shift in the supplied dataset manifest.
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from .benchmark import benchmark
from .benchmark import parser as benchmark_parser
from .experiments import ExperimentConfig, run_experiments
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", required=True)
parser.add_argument("--output", required=True)
parser.add_argument("--samples", type=int, default=512)
parser.add_argument("--device", choices=("auto", "cpu", "mps"), default="auto")
args = parser.parse_args(argv)
content = json.loads(Path(args.config).read_text())
records = content["experiments"] if isinstance(content, dict) else content
configs = [ExperimentConfig(**record) for record in records]
output = Path(args.output)
output.mkdir(parents=True, exist_ok=True)
def evaluate(config):
if Path(config.name).name != config.name or config.name in (".", ".."):
raise ValueError("Experiment names must be a single safe directory name")
parsed = benchmark_parser().parse_args(
[
"--data",
config.data,
"--checkpoint",
config.checkpoint,
"--output",
str(output / config.name),
"--samples",
str(args.samples),
"--device",
args.device,
]
)
parsed.experiment = config
result = benchmark(parsed)
return {"benchmark": f"{config.name}/benchmark.json", "samples": result["samples"]}
run_experiments(configs, evaluate, output / "experiments.json")
if __name__ == "__main__":
main()
|