Download scripts/data/manage_source_split_artifacts.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 15.4 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/manage_source_split_artifacts.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/data/manage_source_split_artifacts.py
-
curl -L -o manage_source_split_artifacts.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/manage_source_split_artifacts.py
15.4 kB
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| import shutil | |
| import time | |
| from pathlib import Path, PurePosixPath | |
| from typing import Any | |
| MANIFEST_NAME = "artifact_manifest.json" | |
| MANIFEST_VERSION = 1 | |
| COPY_CHUNK_BYTES = 16 * 1024 * 1024 | |
| EPHEMERAL_ROOTS = (Path("/dev/shm"), Path("/tmp")) | |
| EPHEMERAL_MARKER_NAME = ".psi0_source_split_ram_stage.json" | |
| _IGNORED_NAMES = { | |
| MANIFEST_NAME, | |
| "preflight_audit_report.json", | |
| ".artifact_stage_complete.json", | |
| EPHEMERAL_MARKER_NAME, | |
| } | |
| def sha256_file(path: Path) -> str: | |
| digest = hashlib.sha256() | |
| with path.open("rb", buffering=COPY_CHUNK_BYTES) as file: | |
| while chunk := file.read(COPY_CHUNK_BYTES): | |
| digest.update(chunk) | |
| return digest.hexdigest() | |
| def manifest_digest(manifest_path: Path) -> str: | |
| return hashlib.sha256(manifest_path.read_bytes()).hexdigest() | |
| def _safe_relative_path(value: str) -> Path: | |
| pure = PurePosixPath(str(value)) | |
| if pure.is_absolute() or not pure.parts or ".." in pure.parts: | |
| raise ValueError(f"Unsafe artifact path: {value!r}") | |
| return Path(*pure.parts) | |
| def _iter_artifact_files(root: Path): | |
| for path in sorted(root.rglob("*")): | |
| if not path.is_file(): | |
| continue | |
| relative = path.relative_to(root) | |
| if path.name in _IGNORED_NAMES: | |
| continue | |
| if any(part.startswith(".artifact_stage") for part in relative.parts): | |
| continue | |
| if path.name.endswith((".tmp", ".lock")): | |
| raise RuntimeError(f"Refusing to package temporary artifact: {path}") | |
| yield path, relative | |
| def _atomic_json(path: Path, value: dict[str, Any]) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") | |
| try: | |
| with temporary.open("w", encoding="utf-8") as file: | |
| json.dump(value, file, indent=2, sort_keys=True) | |
| file.write("\n") | |
| file.flush() | |
| os.fsync(file.fileno()) | |
| os.replace(temporary, path) | |
| finally: | |
| try: | |
| temporary.unlink() | |
| except FileNotFoundError: | |
| pass | |
| def build_manifest(root: Path) -> Path: | |
| root = root.resolve() | |
| if not root.is_dir(): | |
| raise FileNotFoundError(f"Artifact root does not exist: {root}") | |
| started = time.perf_counter() | |
| files: list[dict[str, Any]] = [] | |
| total_bytes = 0 | |
| for index, (path, relative) in enumerate(_iter_artifact_files(root), start=1): | |
| stat = path.stat() | |
| size = int(stat.st_size) | |
| files.append( | |
| { | |
| "path": relative.as_posix(), | |
| "size": size, | |
| "mtime_ns": int(stat.st_mtime_ns), | |
| "sha256": sha256_file(path), | |
| } | |
| ) | |
| total_bytes += size | |
| print( | |
| f"[manifest] file={index} path={relative} size={size / 1024**3:.3f}GiB", | |
| flush=True, | |
| ) | |
| if not files: | |
| raise RuntimeError(f"No artifacts found under {root}") | |
| manifest = { | |
| "version": MANIFEST_VERSION, | |
| "source_split_version": 1, | |
| "files": files, | |
| "file_count": len(files), | |
| "total_bytes": total_bytes, | |
| } | |
| manifest_path = root / MANIFEST_NAME | |
| _atomic_json(manifest_path, manifest) | |
| elapsed = time.perf_counter() - started | |
| print( | |
| f"[manifest ready] files={len(files)} size={total_bytes / 1024**3:.3f}GiB " | |
| f"elapsed={elapsed:.1f}s sha256={manifest_digest(manifest_path)} path={manifest_path}", | |
| flush=True, | |
| ) | |
| return manifest_path | |
| def load_manifest(root: Path) -> tuple[Path, dict[str, Any]]: | |
| manifest_path = root / MANIFEST_NAME | |
| if not manifest_path.is_file(): | |
| raise FileNotFoundError(f"Artifact manifest does not exist: {manifest_path}") | |
| manifest = json.loads(manifest_path.read_text(encoding="utf-8")) | |
| if int(manifest.get("version", 0)) != MANIFEST_VERSION: | |
| raise RuntimeError(f"Unsupported artifact manifest version: {manifest_path}") | |
| files = manifest.get("files") | |
| if not isinstance(files, list) or len(files) != int(manifest.get("file_count", -1)): | |
| raise RuntimeError(f"Malformed artifact manifest: {manifest_path}") | |
| paths = [str(entry.get("path", "")) for entry in files] | |
| if len(set(paths)) != len(paths): | |
| raise RuntimeError(f"Duplicate paths in artifact manifest: {manifest_path}") | |
| for value in paths: | |
| _safe_relative_path(value) | |
| expected_total = sum(int(entry["size"]) for entry in files) | |
| if expected_total != int(manifest.get("total_bytes", -1)): | |
| raise RuntimeError(f"Artifact manifest byte count mismatch: {manifest_path}") | |
| return manifest_path, manifest | |
| def verify_artifacts(root: Path) -> None: | |
| root = root.resolve() | |
| manifest_path, manifest = load_manifest(root) | |
| started = time.perf_counter() | |
| checked_bytes = 0 | |
| for index, entry in enumerate(manifest["files"], start=1): | |
| relative = _safe_relative_path(entry["path"]) | |
| path = root / relative | |
| expected_size = int(entry["size"]) | |
| if not path.is_file() or path.stat().st_size != expected_size: | |
| raise RuntimeError(f"Missing or truncated artifact: {path}") | |
| actual_sha256 = sha256_file(path) | |
| if actual_sha256 != str(entry["sha256"]): | |
| raise RuntimeError(f"SHA256 mismatch for artifact: {path}") | |
| checked_bytes += expected_size | |
| print(f"[verify] file={index}/{len(manifest['files'])} path={relative}", flush=True) | |
| elapsed = time.perf_counter() - started | |
| print( | |
| f"[verify complete] files={len(manifest['files'])} " | |
| f"size={checked_bytes / 1024**3:.3f}GiB elapsed={elapsed:.1f}s " | |
| f"rate={checked_bytes / 1024**3 / max(elapsed, 1e-9):.3f}GiB/s " | |
| f"manifest_sha256={manifest_digest(manifest_path)}", | |
| flush=True, | |
| ) | |
| class StageLock: | |
| def __init__(self, destination: Path, stale_seconds: int = 6 * 3600) -> None: | |
| self.path = destination / ".artifact_stage.lock" | |
| self.stale_seconds = int(stale_seconds) | |
| self.acquired = False | |
| def __enter__(self): | |
| last_message = 0.0 | |
| while True: | |
| try: | |
| self.path.mkdir() | |
| self.acquired = True | |
| return self | |
| except FileExistsError: | |
| try: | |
| age = time.time() - self.path.stat().st_mtime | |
| if age > self.stale_seconds: | |
| self.path.rmdir() | |
| continue | |
| except FileNotFoundError: | |
| continue | |
| now = time.time() | |
| if now - last_message >= 30.0: | |
| print(f"[stage] waiting for lock: {self.path}", flush=True) | |
| last_message = now | |
| time.sleep(1.0) | |
| def __exit__(self, exc_type, exc, traceback): | |
| if self.acquired: | |
| try: | |
| self.path.rmdir() | |
| except FileNotFoundError: | |
| pass | |
| def _copy_verified(source: Path, destination: Path, entry: dict[str, Any]) -> int: | |
| expected_size = int(entry["size"]) | |
| expected_sha256 = str(entry["sha256"]) | |
| if not source.is_file() or source.stat().st_size != expected_size: | |
| raise RuntimeError(f"Missing or truncated source artifact: {source}") | |
| destination.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = destination.with_name(f".{destination.name}.{os.getpid()}.tmp") | |
| digest = hashlib.sha256() | |
| copied = 0 | |
| try: | |
| with source.open("rb", buffering=COPY_CHUNK_BYTES) as source_file, temporary.open( | |
| "wb", buffering=COPY_CHUNK_BYTES | |
| ) as destination_file: | |
| while chunk := source_file.read(COPY_CHUNK_BYTES): | |
| destination_file.write(chunk) | |
| digest.update(chunk) | |
| copied += len(chunk) | |
| destination_file.flush() | |
| os.fsync(destination_file.fileno()) | |
| if copied != expected_size or digest.hexdigest() != expected_sha256: | |
| raise RuntimeError(f"Source artifact failed checksum during copy: {source}") | |
| os.replace(temporary, destination) | |
| os.utime(destination, ns=(int(entry["mtime_ns"]), int(entry["mtime_ns"]))) | |
| finally: | |
| try: | |
| temporary.unlink() | |
| except FileNotFoundError: | |
| pass | |
| return copied | |
| def _require_ephemeral_destination(destination_root: Path) -> Path: | |
| destination_root = destination_root.resolve() | |
| for configured_root in EPHEMERAL_ROOTS: | |
| ephemeral_root = configured_root.resolve() | |
| try: | |
| relative = destination_root.relative_to(ephemeral_root) | |
| except ValueError: | |
| continue | |
| if not relative.parts: | |
| raise ValueError(f"Refusing to use the ephemeral filesystem root: {destination_root}") | |
| return destination_root | |
| roots = ", ".join(str(path.resolve()) for path in EPHEMERAL_ROOTS) | |
| raise ValueError( | |
| f"Ephemeral artifact destination must be under one of [{roots}]: {destination_root}" | |
| ) | |
| def stage_artifacts( | |
| source_root: Path, | |
| destination_root: Path, | |
| *, | |
| ephemeral: bool = False, | |
| ) -> None: | |
| source_root = source_root.resolve() | |
| destination_root = destination_root.resolve() | |
| if ephemeral: | |
| destination_root = _require_ephemeral_destination(destination_root) | |
| source_manifest_path, manifest = load_manifest(source_root) | |
| destination_root.mkdir(parents=True, exist_ok=True) | |
| started = time.perf_counter() | |
| copied_files = copied_bytes = reused_files = 0 | |
| with StageLock(destination_root): | |
| if ephemeral: | |
| free_bytes = int(shutil.disk_usage(destination_root).free) | |
| payload_bytes = int(manifest["total_bytes"]) | |
| required_bytes = payload_bytes + 1024**3 | |
| _atomic_json( | |
| destination_root / EPHEMERAL_MARKER_NAME, | |
| { | |
| "version": 1, | |
| "destination": str(destination_root), | |
| "source": str(source_root), | |
| "pid": os.getpid(), | |
| }, | |
| ) | |
| if free_bytes < required_bytes: | |
| raise RuntimeError( | |
| f"Insufficient ephemeral space for source-split artifacts: " | |
| f"payload={payload_bytes / 1024**3:.3f}GiB " | |
| f"required_with_margin={required_bytes / 1024**3:.3f}GiB " | |
| f"free={free_bytes / 1024**3:.3f}GiB destination={destination_root}" | |
| ) | |
| print( | |
| f"[ephemeral stage] payload={payload_bytes / 1024**3:.3f}GiB " | |
| f"required_with_margin={required_bytes / 1024**3:.3f}GiB " | |
| f"free={free_bytes / 1024**3:.3f}GiB destination={destination_root}", | |
| flush=True, | |
| ) | |
| for entry in manifest["files"]: | |
| relative = _safe_relative_path(entry["path"]) | |
| source = source_root / relative | |
| destination = destination_root / relative | |
| expected_size = int(entry["size"]) | |
| expected_mtime_ns = int(entry["mtime_ns"]) | |
| reusable = ( | |
| destination.is_file() | |
| and destination.stat().st_size == expected_size | |
| and destination.stat().st_mtime_ns == expected_mtime_ns | |
| ) | |
| if not reusable and destination.is_file() and destination.stat().st_size == expected_size: | |
| if sha256_file(destination) == str(entry["sha256"]): | |
| os.utime(destination, ns=(expected_mtime_ns, expected_mtime_ns)) | |
| reusable = True | |
| if reusable: | |
| reused_files += 1 | |
| continue | |
| copied_bytes += _copy_verified(source, destination, entry) | |
| copied_files += 1 | |
| print(f"[stage copied] {relative} ({expected_size / 1024**3:.3f}GiB)", flush=True) | |
| destination_manifest = destination_root / MANIFEST_NAME | |
| if source_manifest_path.resolve() != destination_manifest.resolve(): | |
| temporary_manifest = destination_manifest.with_name( | |
| f".{destination_manifest.name}.{os.getpid()}.tmp" | |
| ) | |
| try: | |
| shutil.copyfile(source_manifest_path, temporary_manifest) | |
| os.replace(temporary_manifest, destination_manifest) | |
| finally: | |
| try: | |
| temporary_manifest.unlink() | |
| except FileNotFoundError: | |
| pass | |
| _atomic_json( | |
| destination_root / ".artifact_stage_complete.json", | |
| { | |
| "version": 1, | |
| "manifest_sha256": manifest_digest(source_manifest_path), | |
| "file_count": len(manifest["files"]), | |
| "total_bytes": int(manifest["total_bytes"]), | |
| }, | |
| ) | |
| elapsed = time.perf_counter() - started | |
| print( | |
| f"[stage ready] source={source_root} destination={destination_root} " | |
| f"files={len(manifest['files'])} reused={reused_files} copied={copied_files} " | |
| f"copied_gib={copied_bytes / 1024**3:.3f} elapsed={elapsed:.1f}s", | |
| flush=True, | |
| ) | |
| def cleanup_ephemeral_artifacts(root: Path) -> None: | |
| root = root.resolve() | |
| if not root.exists(): | |
| print(f"[ephemeral cleanup] already absent: {root}", flush=True) | |
| return | |
| root = _require_ephemeral_destination(root) | |
| marker_path = root / EPHEMERAL_MARKER_NAME | |
| if not marker_path.is_file(): | |
| raise RuntimeError(f"Refusing to remove unowned ephemeral directory: {root}") | |
| marker = json.loads(marker_path.read_text(encoding="utf-8")) | |
| if int(marker.get("version", 0)) != 1 or str(marker.get("destination", "")) != str(root): | |
| raise RuntimeError(f"Invalid ephemeral ownership marker: {marker_path}") | |
| started = time.perf_counter() | |
| shutil.rmtree(root) | |
| print( | |
| f"[ephemeral cleanup] removed={root} elapsed={time.perf_counter() - started:.1f}s", | |
| flush=True, | |
| ) | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description="Build, verify, or stage source-split artifacts.") | |
| subparsers = parser.add_subparsers(dest="command", required=True) | |
| build = subparsers.add_parser("build-manifest") | |
| build.add_argument("--root", type=Path, required=True) | |
| verify = subparsers.add_parser("verify") | |
| verify.add_argument("--root", type=Path, required=True) | |
| stage = subparsers.add_parser("stage") | |
| stage.add_argument("--source", type=Path, required=True) | |
| stage.add_argument("--destination", type=Path, required=True) | |
| stage.add_argument("--ephemeral", action="store_true") | |
| cleanup = subparsers.add_parser("cleanup-ephemeral") | |
| cleanup.add_argument("--root", type=Path, required=True) | |
| return parser.parse_args() | |
| def main() -> None: | |
| args = parse_args() | |
| if args.command == "build-manifest": | |
| build_manifest(args.root) | |
| elif args.command == "verify": | |
| verify_artifacts(args.root) | |
| elif args.command == "stage": | |
| stage_artifacts(args.source, args.destination, ephemeral=args.ephemeral) | |
| elif args.command == "cleanup-ephemeral": | |
| cleanup_ephemeral_artifacts(args.root) | |
| else: | |
| raise RuntimeError(f"Unhandled command: {args.command}") | |
| if __name__ == "__main__": | |
| main() | |