File size: 2,048 Bytes
1baf215
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
"""Run explicit B/C/D/E experiment configurations without inventing results.

JSON input is a list of ExperimentConfig records or {"experiments": [...]}.
Data/checkpoint paths are mandatory to execute; E checks excluded_families in
run.json and D checks configuration_shift in the supplied dataset manifest.
"""

from __future__ import annotations

import argparse
import json
from pathlib import Path

from .benchmark import benchmark
from .benchmark import parser as benchmark_parser
from .experiments import ExperimentConfig, run_experiments


def main(argv=None):
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--config", required=True)
    parser.add_argument("--output", required=True)
    parser.add_argument("--samples", type=int, default=512)
    parser.add_argument("--device", choices=("auto", "cpu", "mps"), default="auto")
    args = parser.parse_args(argv)
    content = json.loads(Path(args.config).read_text())
    records = content["experiments"] if isinstance(content, dict) else content
    configs = [ExperimentConfig(**record) for record in records]
    output = Path(args.output)
    output.mkdir(parents=True, exist_ok=True)

    def evaluate(config):
        if Path(config.name).name != config.name or config.name in (".", ".."):
            raise ValueError("Experiment names must be a single safe directory name")
        parsed = benchmark_parser().parse_args(
            [
                "--data",
                config.data,
                "--checkpoint",
                config.checkpoint,
                "--output",
                str(output / config.name),
                "--samples",
                str(args.samples),
                "--device",
                args.device,
            ]
        )
        parsed.experiment = config
        result = benchmark(parsed)
        return {"benchmark": f"{config.name}/benchmark.json", "samples": result["samples"]}

    run_experiments(configs, evaluate, output / "experiments.json")


if __name__ == "__main__":
    main()