File size: 1,628 Bytes
cea7f55 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 | """Write synthetic data: data/<name>/{secrets,injection,toolcall}.jsonl.
python scripts/build_eval.py # eval set (eval templates)
python scripts/build_eval.py --split train --n 4000 --seed 1 --out data/train
python scripts/build_eval.py --split train --n 400 --seed 2 --out data/dev
"""
import argparse
import json
import logging
import random
from pathlib import Path
from vela_guard import gen_injection, gen_secrets, gen_toolcall
GENERATORS = {"secrets": gen_secrets, "injection": gen_injection, "toolcall": gen_toolcall}
EVAL_SEED = 20261012
log = logging.getLogger("build_eval")
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--out", type=Path, default=Path("data/eval"))
parser.add_argument("--n", type=int, default=300, help="examples per task")
parser.add_argument("--seed", type=int, default=EVAL_SEED)
parser.add_argument("--split", choices=("eval", "train"), default="eval")
args = parser.parse_args()
args.out.mkdir(parents=True, exist_ok=True)
for offset, (task, module) in enumerate(GENERATORS.items()):
rows = module.generate(random.Random(args.seed + offset), args.n, split=args.split)
path = args.out / f"{task}.jsonl"
path.write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in rows))
log.info("%s: %d rows (%d hard) -> %s", task, len(rows), sum(r["hard"] for r in rows), path)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO, format="%(message)s")
main()
|