File size: 2,634 Bytes
f4cc109 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 | #!/usr/bin/env python3
"""Validate a Task 3 submission before scoring it.
Validation is ID-based: predictions are matched to reference cases by ``id``,
never by row order.
python scripts/validate_submission.py \
--predictions predictions.jsonl \
--reference data/public_dev_inputs.jsonl
Exit codes: 0 = valid, 1 = invalid, 2 = could not run (bad arguments/paths).
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
import _bootstrap # noqa: F401
from task3_core.io import DataError
from task3_core.schema import SchemaError, SubmissionMetadata
from task3_core.validation import validate_submission_file
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--predictions", required=True, help="predictions.jsonl to validate")
parser.add_argument(
"--reference",
default="data/public_dev_inputs.jsonl",
help="Inputs or gold file defining the expected case ids",
)
parser.add_argument("--format", dest="fmt", default="native", choices=("native", "lm-eval"))
parser.add_argument(
"--reference-kind", default="auto", choices=("auto", "inputs", "gold"),
help="How to read --reference (default: auto-detect)",
)
parser.add_argument(
"--allow-empty-values", action="store_true",
help="Downgrade empty extracted/calculated values from error to warning",
)
parser.add_argument("--metadata", default=None, help="Optional metadata.json to validate too")
parser.add_argument("--json", dest="json_out", default=None, help="Write the report as JSON")
args = parser.parse_args()
try:
report = validate_submission_file(
args.predictions,
args.reference,
fmt=args.fmt,
reference_kind=args.reference_kind,
allow_empty_values=args.allow_empty_values,
)
except DataError as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 2
if args.metadata:
try:
SubmissionMetadata.from_dict(json.loads(Path(args.metadata).read_text(encoding="utf-8")))
except (OSError, json.JSONDecodeError, SchemaError) as exc:
report.errors.append(f"metadata.json: {exc}")
print(report.render())
if args.json_out:
Path(args.json_out).write_text(json.dumps(report.to_dict(), indent=2) + "\n", encoding="utf-8")
print(f"\nWrote {args.json_out}")
return 0 if report.ok else 1
if __name__ == "__main__":
sys.exit(main())
|