File size: 2,030 Bytes
d699ea7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | #!/usr/bin/env python3
"""GAP-01: audit that no held-out validation day leaked into training.
Cross-checks ``configs/operating.yml``:
- ``validation.train_days`` (inputs to stage3_tcn_prod.pt)
- ``validation.days`` (the held-out 6-day validation table)
Fails (exit 1) if any held-out day appears in the training set at the (symbol,
date) granularity, or if a training-input window file is missing from
``--windows-dir``.
Usage:
python scripts/audit_dataset_split.py --config configs/operating.yml --windows-dir data/windows
"""
import argparse
import sys
from pathlib import Path
import yaml
def main() -> int:
ap = argparse.ArgumentParser(description="GAP-01 train/test day-split audit")
ap.add_argument("--config", default="configs/operating.yml")
ap.add_argument("--windows-dir", default="data/windows",
help="dir to check training window files exist")
args = ap.parse_args()
with open(args.config, encoding="utf-8") as f:
op = yaml.safe_load(f)
val = op["validation"]
train = val.get("train_days", [])
held = val.get("days", [])
train_keys = {(d["symbol"], d["date"]) for d in train}
held_keys = {(d["symbol"], d["date"]) for d in held}
overlap = train_keys & held_keys
problems = []
if overlap:
problems.append(f"LEAK: held-out day(s) present in training set: {sorted(overlap)}")
else:
print(f"OK: {len(train)} train days and {len(held)} held-out days are disjoint at (symbol, date).")
# Training-input window files must exist.
win_dir = Path(args.windows_dir)
for d in train:
f = d.get("file")
if f and not (win_dir / f).exists():
problems.append(f"MISSING training window file: {win_dir / f}")
if problems:
for p in problems:
print("ERROR:", p, file=sys.stderr)
return 1
print(f"OK: all {len(train)} training window files exist under {win_dir}.")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|