Buckets:

download
raw
16.7 kB
#!/usr/bin/env python3
"""Execute a matched StateSet algorithm benchmark and emit strict evidence.
Each algorithm adapter runs as a separate process. This orchestrator rotates
execution order by seed, measures wall time outside the adapter, verifies both
the shared protocol and algorithm-specific configuration attestations, hashes
the normalized policy artifact, and emits documents accepted by
``algorithm_comparison.py``.
"""
from __future__ import annotations
import argparse
import json
import os
import shlex
import subprocess
import sys
import time
from collections.abc import Mapping, Sequence
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from framework_comparison import EvidenceError, validate_document
from shootout import (
ShootoutError,
_format_command,
canonical_digest,
git_commit,
hash_artifact,
)
COMMON_MANIFEST_FIELDS = (
"protocol",
"model",
"model_revision",
"task",
"dataset_revision",
"cache_policy",
)
REQUIRED_COMMAND_PLACEHOLDERS = {
"{algorithm}",
"{seed}",
"{adapter_output}",
"{artifact_dir}",
"{phase0_output}",
"{model}",
"{model_revision}",
"{dataset_revision}",
"{task}",
"{config_json}",
"{num_train_examples}",
"{num_eval_examples}",
}
def _nonempty_string(value: Any, field: str) -> str:
if not isinstance(value, str) or not value.strip():
raise ShootoutError(f"{field} must be a non-empty string")
return value
def load_manifest(path: Path) -> dict[str, Any]:
"""Load a strict algorithm-shootout manifest."""
try:
raw = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise ShootoutError(f"{path}: invalid JSON") from exc
if not isinstance(raw, Mapping) or raw.get("schema_version") != 1:
raise ShootoutError("manifest must be an object with schema_version=1")
for field in COMMON_MANIFEST_FIELDS:
_nonempty_string(raw.get(field), f"manifest.{field}")
for field in ("model_revision", "dataset_revision"):
if len(str(raw[field])) != 40:
raise ShootoutError(f"manifest.{field} must be a full commit SHA")
framework = raw.get("framework")
if not isinstance(framework, Mapping):
raise ShootoutError("manifest.framework must be an object")
for field in ("name", "version"):
_nonempty_string(framework.get(field), f"manifest.framework.{field}")
config = raw.get("config")
if not isinstance(config, Mapping) or not config:
raise ShootoutError("manifest.config must be a non-empty object")
for field in (
"num_train_examples",
"num_eval_examples",
"max_steps",
"per_device_train_batch_size",
"gradient_accumulation_steps",
"num_generations",
"num_iterations",
):
value = config.get(field)
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
raise ShootoutError(f"manifest.config.{field} must be an integer >= 1")
# Several native objectives expose gradient accumulation in their config
# for compatibility but do not all implement the same accumulation
# semantics. Refuse an allegedly matched matrix until that is true.
if config["gradient_accumulation_steps"] != 1:
raise ShootoutError(
"algorithm shootouts currently require gradient_accumulation_steps=1"
)
seeds = raw.get("seeds")
if (
not isinstance(seeds, list)
or len(seeds) < 3
or any(
isinstance(seed, bool) or not isinstance(seed, int) or seed < 0
for seed in seeds
)
or len(seeds) != len(set(seeds))
):
raise ShootoutError(
"manifest.seeds must contain at least three unique non-negative integers"
)
hardware = raw.get("hardware")
if not isinstance(hardware, Mapping):
raise ShootoutError("manifest.hardware must be an object")
_nonempty_string(hardware.get("gpu"), "manifest.hardware.gpu")
_nonempty_string(hardware.get("cuda"), "manifest.hardware.cuda")
gpu_count = hardware.get("gpu_count")
if isinstance(gpu_count, bool) or not isinstance(gpu_count, int) or gpu_count < 1:
raise ShootoutError("manifest.hardware.gpu_count must be >= 1")
algorithms = raw.get("algorithms")
if not isinstance(algorithms, list) or len(algorithms) < 2:
raise ShootoutError("manifest requires at least two algorithms")
names: list[str] = []
for algorithm in algorithms:
if not isinstance(algorithm, Mapping):
raise ShootoutError("each algorithm must be an object")
name = _nonempty_string(algorithm.get("name"), "algorithm.name")
_nonempty_string(algorithm.get("revision"), f"algorithm[{name}].revision")
algorithm_config = algorithm.get("config")
if not isinstance(algorithm_config, Mapping) or not algorithm_config:
raise ShootoutError(f"algorithm[{name}].config must be a non-empty object")
if not isinstance(algorithm_config.get("objective"), str):
raise ShootoutError(f"algorithm[{name}].config.objective must be a string")
command = algorithm.get("command")
if (
not isinstance(command, list)
or not command
or any(not isinstance(part, str) or not part for part in command)
):
raise ShootoutError(
f"algorithm[{name}].command must be a non-empty string list"
)
missing = REQUIRED_COMMAND_PLACEHOLDERS.difference(command)
if missing:
raise ShootoutError(
f"algorithm[{name}].command is missing placeholders: "
+ ", ".join(sorted(missing))
)
names.append(name)
if len(names) != len(set(names)):
raise ShootoutError("algorithm names must be unique")
return dict(raw)
def execution_order(
algorithms: Sequence[Mapping[str, Any]], seed_index: int
) -> list[Mapping[str, Any]]:
"""Rotate algorithm order each seed to reduce thermal/cache order bias."""
offset = seed_index % len(algorithms)
return [*algorithms[offset:], *algorithms[:offset]]
def validate_adapter_result(
raw: Mapping[str, Any],
manifest: Mapping[str, Any],
algorithm: Mapping[str, Any],
source: Path,
) -> dict[str, Any]:
"""Validate Phase-0's neutral result and both config attestations."""
if raw.get("status") != "completed" or raw.get("measured") is not True:
raise ShootoutError(f"{source}: adapter did not report measured completion")
if raw.get("framework_version") != manifest["framework"]["version"]:
raise ShootoutError(f"{source}: framework version does not match manifest")
if raw.get("config_sha256") != canonical_digest(manifest["config"]):
raise ShootoutError(f"{source}: shared config digest does not match manifest")
algorithm_config = raw.get("algorithm_config")
if algorithm_config != algorithm["config"]:
raise ShootoutError(f"{source}: algorithm config does not match manifest")
if raw.get("algorithm_config_sha256") != canonical_digest(algorithm["config"]):
raise ShootoutError(
f"{source}: algorithm config digest does not match manifest"
)
artifact_path = raw.get("artifact_path")
if not isinstance(artifact_path, str) or not artifact_path:
raise ShootoutError(f"{source}: artifact_path must be non-empty")
hardware = raw.get("hardware")
if not isinstance(hardware, Mapping):
raise ShootoutError(f"{source}: hardware must be an object")
for field in ("gpu", "gpu_count", "cuda"):
if hardware.get(field) != manifest["hardware"].get(field):
raise ShootoutError(f"{source}: hardware.{field} does not match manifest")
metrics = raw.get("metrics")
if not isinstance(metrics, Mapping):
raise ShootoutError(f"{source}: metrics must be an object")
for field in (
"samples_processed",
"peak_vram_mb",
"eval_score_baseline",
"eval_score_final",
):
value = metrics.get(field)
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise ShootoutError(f"{source}: metrics.{field} must be numeric")
if float(metrics["samples_processed"]) <= 0:
raise ShootoutError(f"{source}: samples_processed must be > 0")
if float(metrics["peak_vram_mb"]) <= 0:
raise ShootoutError(f"{source}: peak_vram_mb must be > 0")
return dict(raw)
def run_algorithm(
manifest: Mapping[str, Any],
algorithm: Mapping[str, Any],
seed: int,
output_dir: Path,
root: Path,
timeout_seconds: int,
) -> Path:
"""Run one algorithm/seed and emit one validated evidence document."""
slug = str(algorithm["name"]).lower().replace(" ", "-")
run_dir = output_dir / "runs" / f"{slug}-seed{seed}"
run_dir.mkdir(parents=True, exist_ok=False)
adapter_output = run_dir / "adapter-result.json"
artifact_dir = run_dir / "artifact"
artifact_dir.mkdir()
values = {
"algorithm": algorithm["name"],
"seed": seed,
"adapter_output": str(adapter_output.resolve()),
"phase0_output": str((run_dir / "phase0-result.json").resolve()),
"artifact_dir": str(artifact_dir.resolve()),
"model": manifest["model"],
"model_revision": manifest["model_revision"],
"dataset_revision": manifest["dataset_revision"],
"task": manifest["task"],
"config_json": json.dumps(
manifest["config"], sort_keys=True, separators=(",", ":")
),
"num_train_examples": manifest["config"]["num_train_examples"],
"num_eval_examples": manifest["config"]["num_eval_examples"],
}
command = _format_command(algorithm["command"], values)
started = time.monotonic()
try:
completed = subprocess.run(
command,
cwd=root,
capture_output=True,
text=True,
check=False,
timeout=timeout_seconds,
env=os.environ.copy(),
)
elapsed = time.monotonic() - started
except subprocess.TimeoutExpired as exc:
elapsed = time.monotonic() - started
(run_dir / "failure.json").write_text(
json.dumps(
{"kind": "timeout", "elapsed_seconds": elapsed, "command": command},
indent=2,
)
+ "\n",
encoding="utf-8",
)
raise ShootoutError(f"{algorithm['name']} seed {seed} timed out") from exc
(run_dir / "stdout.log").write_text(completed.stdout, encoding="utf-8")
(run_dir / "stderr.log").write_text(completed.stderr, encoding="utf-8")
if completed.returncode != 0:
(run_dir / "failure.json").write_text(
json.dumps(
{
"kind": "exit",
"returncode": completed.returncode,
"command": command,
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
raise ShootoutError(
f"{algorithm['name']} seed {seed} exited {completed.returncode}"
)
if not adapter_output.is_file():
raise ShootoutError(f"{algorithm['name']} seed {seed} wrote no adapter result")
try:
raw = json.loads(adapter_output.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise ShootoutError(f"{adapter_output}: invalid JSON") from exc
if not isinstance(raw, Mapping):
raise ShootoutError(f"{adapter_output}: result must be an object")
adapter = validate_adapter_result(raw, manifest, algorithm, adapter_output)
artifact_path = Path(adapter["artifact_path"])
if not artifact_path.is_absolute():
artifact_path = root / artifact_path
artifact_path = artifact_path.resolve()
if not artifact_path.is_relative_to(artifact_dir.resolve()):
raise ShootoutError("adapter artifact_path must stay inside {artifact_dir}")
samples = float(adapter["metrics"]["samples_processed"])
evidence = {
"schema_version": 1,
"measured": True,
"framework": manifest["framework"]["name"],
"framework_version": manifest["framework"]["version"],
"harness_commit": git_commit(root),
"protocol": manifest["protocol"],
"cache_policy": manifest["cache_policy"],
"algorithm": algorithm["name"],
"algorithm_revision": algorithm["revision"],
"model": manifest["model"],
"model_revision": manifest["model_revision"],
"task": manifest["task"],
"dataset_revision": manifest["dataset_revision"],
"seed": seed,
"timestamp": datetime.now(timezone.utc).isoformat(),
"command": shlex.join(command),
"config": {
"shared": dict(manifest["config"]),
"algorithm": dict(algorithm["config"]),
},
"hardware": dict(adapter["hardware"]),
"metrics": {
"samples_per_second": samples / elapsed,
"wall_clock_seconds": elapsed,
"peak_vram_mb": float(adapter["metrics"]["peak_vram_mb"]),
"eval_score_baseline": float(adapter["metrics"]["eval_score_baseline"]),
"eval_score_final": float(adapter["metrics"]["eval_score_final"]),
},
"artifact_sha256": hash_artifact(artifact_path),
}
destination = output_dir / f"{slug}-seed{seed}.json"
validate_document(evidence, destination)
destination.write_text(json.dumps(evidence, indent=2) + "\n", encoding="utf-8")
return destination
def main(argv: Sequence[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("manifest", type=Path)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--root", type=Path, default=Path.cwd())
parser.add_argument("--timeout-seconds", type=int, default=14400)
parser.add_argument(
"--required-algorithm",
action="append",
default=[],
help="Algorithm required in the manifest (repeatable).",
)
parser.add_argument(
"--preflight",
action="store_true",
help="Run only the first seed across every algorithm as non-publishable diagnostics.",
)
parser.add_argument("--dry-run", action="store_true")
args = parser.parse_args(argv)
try:
manifest = load_manifest(args.manifest)
if args.timeout_seconds < 1:
raise ShootoutError("timeout-seconds must be >= 1")
names = {str(algorithm["name"]) for algorithm in manifest["algorithms"]}
missing = sorted(set(args.required_algorithm) - names)
if missing:
raise ShootoutError(
"manifest is missing required algorithms: " + ", ".join(missing)
)
seeds = manifest["seeds"][:1] if args.preflight else manifest["seeds"]
if args.dry_run:
for index, seed in enumerate(seeds):
for algorithm in execution_order(manifest["algorithms"], index):
print(f"seed={seed} algorithm={algorithm['name']}")
return 0
args.output_dir.mkdir(parents=True, exist_ok=True)
produced: list[Path] = []
failures: list[str] = []
for index, seed in enumerate(seeds):
for algorithm in execution_order(manifest["algorithms"], index):
try:
produced.append(
run_algorithm(
manifest,
algorithm,
seed,
args.output_dir,
args.root,
args.timeout_seconds,
)
)
except (ShootoutError, EvidenceError) as exc:
failure = f"{algorithm['name']} seed {seed}: {exc}"
failures.append(failure)
print(f"algorithm run failed: {failure}", file=sys.stderr)
if failures:
raise ShootoutError(
f"{len(failures)} of {len(produced) + len(failures)} runs failed; "
"all attempted runs and failure logs were retained"
)
except (ShootoutError, EvidenceError) as exc:
print(f"algorithm shootout rejected: {exc}", file=sys.stderr)
return 2
print(f"wrote {len(produced)} measured algorithm evidence documents")
return 0
if __name__ == "__main__":
raise SystemExit(main())

Xet Storage Details

Size:
16.7 kB
·
Xet hash:
f531265dbad20cf082d9b1a8c30c27164314398665b2035b2002bd6285378e0c

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.