File size: 2,046 Bytes
78ac017 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | """Bind the public protocol template to the user's own evaluation assets."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from experiments.evaluate_qwen3vl_recent4_full import (
sha256_file,
validate_processor_contract,
)
def configure(
*,
template: Path,
streaming_samples: Path,
ovo_samples: Path,
processor_config: Path,
output: Path,
) -> None:
protocol = json.loads(template.read_text(encoding="utf-8"))
for benchmark, path in (
("streamingbench", streaming_samples),
("ovo", ovo_samples),
):
path = path.resolve(strict=True)
protocol["benchmarks"][benchmark]["export"] = str(path)
protocol["benchmarks"][benchmark]["export_sha256"] = sha256_file(path)
processor_config = processor_config.resolve(strict=True)
if processor_config.name != "preprocessor_config.json":
raise ValueError("Pass the processor's preprocessor_config.json")
protocol["inference"]["processor_contract"]["preprocessor_config_sha256"] = (
sha256_file(processor_config)
)
validate_processor_contract(processor_config.parent, protocol)
protocol["status"] = "locked"
output.parent.mkdir(parents=True, exist_ok=True)
with output.open("x", encoding="utf-8") as handle:
json.dump(protocol, handle, indent=2)
handle.write("\n")
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--template",
type=Path,
default=Path("configs/evaluation/recent4.example.json"),
)
parser.add_argument("--streaming-samples", type=Path, required=True)
parser.add_argument("--ovo-samples", type=Path, required=True)
parser.add_argument("--processor-config", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
configure(**vars(args))
print(f"Created local evaluation protocol: {args.output}")
if __name__ == "__main__":
main()
|