| |
| """Stage an existing local VideoMAE-v2 Large snapshot; no network access.""" |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import hashlib |
| import json |
| import os |
| from pathlib import Path |
| import tempfile |
| import shutil |
|
|
|
|
| MODEL_ID = "VideoMAEv2-Large" |
| REVISION = "9981a9c8f77118c421e5228e1b219468a4b0238d" |
|
|
|
|
| def file_sha256(path: Path) -> str: |
| digest = hashlib.sha256() |
| with path.open("rb") as handle: |
| while block := handle.read(8 * 1024 * 1024): |
| digest.update(block) |
| return digest.hexdigest() |
|
|
|
|
| def canonical_sha256(value) -> str: |
| payload = json.dumps( |
| value, sort_keys=True, separators=(",", ":"), allow_nan=False |
| ).encode("utf-8") |
| return hashlib.sha256(payload).hexdigest() |
|
|
|
|
| def atomic_json(path: Path, value) -> None: |
| path.parent.mkdir(parents=True, exist_ok=True) |
| descriptor, name = tempfile.mkstemp( |
| dir=path.parent, prefix=f".{path.name}.", suffix=".tmp") |
| try: |
| with os.fdopen(descriptor, "w", encoding="utf-8") as handle: |
| json.dump(value, handle, sort_keys=True, indent=2) |
| handle.write("\n") |
| handle.flush() |
| os.fsync(handle.fileno()) |
| os.replace(name, path) |
| finally: |
| Path(name).unlink(missing_ok=True) |
|
|
|
|
| def main() -> None: |
| parser = argparse.ArgumentParser() |
| parser.add_argument( |
| "--output-dir", |
| default=( |
| "assets/models/" |
| "VideoMAEv2-Large"), |
| ) |
| parser.add_argument("--source-dir", required=True, type=Path, |
| help="Existing local snapshot; not a hosted repository ID") |
| args = parser.parse_args() |
| output = Path(args.output_dir).expanduser().resolve() |
| source = args.source_dir.expanduser().resolve() |
| if not source.is_dir(): |
| parser.error("--source-dir must be an existing local directory") |
| if source == output or source in output.parents or output in source.parents: |
| parser.error("Source and output directories must be separate and non-nested") |
| if output.exists() and (not output.is_dir() or any(output.iterdir())): |
| parser.error("--output-dir must be absent or empty") |
| if not (source / "config.json").is_file(): |
| parser.error("The local snapshot must include config.json") |
| if not any(source.glob("*.safetensors")) and not any(source.glob("*.bin")): |
| parser.error("The local snapshot must include model weights") |
| shutil.copytree(source, output, dirs_exist_ok=True, |
| ignore=shutil.ignore_patterns(".git", ".cache", "__pycache__", ".DS_Store")) |
| |
| |
| existing_manifest = output / "download_manifest.json" |
| if existing_manifest.is_file(): |
| manifest = json.loads(existing_manifest.read_text()) |
| payload = dict(manifest) |
| saved = payload.pop("content_hash", None) |
| if manifest.get("revision") != REVISION or canonical_sha256(payload) != saved: |
| raise ValueError("Invalid pinned local snapshot manifest") |
| for item in manifest["files"]: |
| path = (output / item["path"]).resolve() |
| if (output not in path.parents or path.stat().st_size != item["size_bytes"] |
| or file_sha256(path) != item["sha256"]): |
| raise ValueError(f"Local snapshot file identity mismatch: {path}") |
| print(json.dumps({"output_dir": str(output), "file_count": len(manifest["files"]), |
| "content_hash": saved, "preserved_manifest": True}, sort_keys=True)) |
| return |
| files = [] |
| for path in sorted(output.rglob("*")): |
| if ( |
| not path.is_file() |
| or path.name == "download_manifest.json" |
| or ".cache" in path.parts |
| ): |
| continue |
| files.append({ |
| "path": path.relative_to(output).as_posix(), |
| "size_bytes": path.stat().st_size, |
| "sha256": file_sha256(path), |
| }) |
| payload = { |
| "schema_version": 1, |
| "model_id": MODEL_ID, |
| "revision": REVISION, |
| "files": files, |
| } |
| payload["content_hash"] = canonical_sha256(payload) |
| atomic_json(output / "download_manifest.json", payload) |
| print(json.dumps({ |
| "output_dir": str(output), |
| "file_count": len(files), |
| "content_hash": payload["content_hash"], |
| }, sort_keys=True)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
