File size: 1,576 Bytes
dfb775d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | """Dataset verification — recompute BLAKE3 over the local files referenced by
a shard manifest, surface mismatches.
Pure stdlib + `mindxtrain.provenance.hashing`.
"""
from __future__ import annotations
import json
from pathlib import Path
from pydantic import BaseModel, ConfigDict, Field
from mindxtrain.provenance.hashing import blake3_file
class VerifyResult(BaseModel):
model_config = ConfigDict(extra="forbid")
manifest_path: Path
matched: int = 0
mismatched: list[str] = Field(default_factory=list)
missing: list[str] = Field(default_factory=list)
def verify_dataset(manifest_path: Path, root: Path) -> VerifyResult:
"""Walk the manifest, recompute hashes, return mismatches.
Manifest format expected:
{"shards": [{"path": "shard-00000.tar", "blake3": "...."}, ...]}
`path` is resolved relative to `root`.
"""
manifest_path = Path(manifest_path)
root = Path(root)
raw = json.loads(manifest_path.read_text())
shards = raw.get("shards") or []
result = VerifyResult(manifest_path=manifest_path)
for shard in shards:
rel = shard.get("path", "")
expected = shard.get("blake3", "")
if not rel or not expected:
continue
local = root / rel
if not local.exists():
result.missing.append(rel)
continue
actual = blake3_file(local)
if actual != expected:
result.mismatched.append(rel)
else:
result.matched += 1
return result
__all__ = ["VerifyResult", "verify_dataset"]
|