| |
| """Verify that frozen pair/action manifests resolve against reconstructed data.""" |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import hashlib |
| import json |
| from pathlib import Path |
|
|
|
|
| def digest(path: Path) -> str: |
| result = hashlib.sha256() |
| with path.open("rb") as stream: |
| for block in iter(lambda: stream.read(8 * 1024 * 1024), b""): |
| result.update(block) |
| return result.hexdigest() |
|
|
|
|
| def main() -> None: |
| parser = argparse.ArgumentParser() |
| parser.add_argument("--manifest", type=Path, required=True) |
| parser.add_argument("--processed-root", type=Path, required=True) |
| parser.add_argument("--checksums", type=Path) |
| parser.add_argument("--action-manifest-dir", type=Path) |
| parser.add_argument("--derived-root", type=Path) |
| args = parser.parse_args() |
|
|
| rows = [json.loads(line) for line in args.manifest.read_text().splitlines() if line] |
| references: set[Path] = set() |
| for row in rows: |
| for role in ("chosen", "rejected"): |
| relative = Path(str(row[role]["source_frame_file"])) |
| parts = relative.parts[1:] if relative.parts[:1] in ((".",), ("processed_datasets",)) else relative.parts |
| if relative.parts[:1] == (".",) and parts[:1] == ("processed_datasets",): |
| parts = parts[1:] |
| references.add(Path(*parts)) |
| missing = [str(path) for path in sorted(references) if not (args.processed_root / path).is_file()] |
| if missing: |
| raise FileNotFoundError(f"missing {len(missing)} manifest frame files; examples={missing[:10]}") |
| if len(rows) != 2689 or len(references) != 4323: |
| raise RuntimeError(f"unexpected frozen counts: rows={len(rows)} files={len(references)}") |
|
|
| checksum_count = 0 |
| if args.checksums: |
| for line in args.checksums.read_text().splitlines(): |
| expected, relative = line.split(maxsplit=1) |
| relative_path = Path(relative.strip()) |
| parts = relative_path.parts[1:] if relative_path.parts[:1] == ("processed_datasets",) else relative_path.parts |
| target = args.processed_root / Path(*parts) |
| if digest(target) != expected: |
| raise RuntimeError(f"SHA256 mismatch: {target}") |
| checksum_count += 1 |
|
|
| action_rows = action_files = 0 |
| if args.action_manifest_dir or args.derived_root: |
| if not (args.action_manifest_dir and args.derived_root): |
| raise ValueError("action-manifest-dir and derived-root must be supplied together") |
| for manifest in sorted(args.action_manifest_dir.glob("*.jsonl")): |
| for line in manifest.read_text().splitlines(): |
| row = json.loads(line) |
| action_rows += 1 |
| if "sidecar_path" not in row: |
| continue |
| target = args.derived_root / row["sidecar_path"] |
| if not target.is_file(): |
| raise FileNotFoundError(target) |
| action_files += 1 |
|
|
| print(json.dumps({ |
| "status": "pass", |
| "pair_rows": len(rows), |
| "unique_stage2_frame_files": len(references), |
| "sha256_files_verified": checksum_count, |
| "action_manifest_rows": action_rows, |
| "action_sidecar_files_resolved": action_files, |
| }, indent=2)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
