File size: 3,339 Bytes
91ddf7c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | #!/usr/bin/env python3
"""Verify that frozen pair/action manifests resolve against reconstructed data."""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
def digest(path: Path) -> str:
result = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(8 * 1024 * 1024), b""):
result.update(block)
return result.hexdigest()
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--processed-root", type=Path, required=True)
parser.add_argument("--checksums", type=Path)
parser.add_argument("--action-manifest-dir", type=Path)
parser.add_argument("--derived-root", type=Path)
args = parser.parse_args()
rows = [json.loads(line) for line in args.manifest.read_text().splitlines() if line]
references: set[Path] = set()
for row in rows:
for role in ("chosen", "rejected"):
relative = Path(str(row[role]["source_frame_file"]))
parts = relative.parts[1:] if relative.parts[:1] in ((".",), ("processed_datasets",)) else relative.parts
if relative.parts[:1] == (".",) and parts[:1] == ("processed_datasets",):
parts = parts[1:]
references.add(Path(*parts))
missing = [str(path) for path in sorted(references) if not (args.processed_root / path).is_file()]
if missing:
raise FileNotFoundError(f"missing {len(missing)} manifest frame files; examples={missing[:10]}")
if len(rows) != 2689 or len(references) != 4323:
raise RuntimeError(f"unexpected frozen counts: rows={len(rows)} files={len(references)}")
checksum_count = 0
if args.checksums:
for line in args.checksums.read_text().splitlines():
expected, relative = line.split(maxsplit=1)
relative_path = Path(relative.strip())
parts = relative_path.parts[1:] if relative_path.parts[:1] == ("processed_datasets",) else relative_path.parts
target = args.processed_root / Path(*parts)
if digest(target) != expected:
raise RuntimeError(f"SHA256 mismatch: {target}")
checksum_count += 1
action_rows = action_files = 0
if args.action_manifest_dir or args.derived_root:
if not (args.action_manifest_dir and args.derived_root):
raise ValueError("action-manifest-dir and derived-root must be supplied together")
for manifest in sorted(args.action_manifest_dir.glob("*.jsonl")):
for line in manifest.read_text().splitlines():
row = json.loads(line)
action_rows += 1
if "sidecar_path" not in row:
continue
target = args.derived_root / row["sidecar_path"]
if not target.is_file():
raise FileNotFoundError(target)
action_files += 1
print(json.dumps({
"status": "pass",
"pair_rows": len(rows),
"unique_stage2_frame_files": len(references),
"sha256_files_verified": checksum_count,
"action_manifest_rows": action_rows,
"action_sidecar_files_resolved": action_files,
}, indent=2))
if __name__ == "__main__":
main()
|