from __future__ import annotations import argparse import hashlib import json import os import shutil import time from pathlib import Path, PurePosixPath from typing import Any MANIFEST_NAME = "artifact_manifest.json" MANIFEST_VERSION = 1 COPY_CHUNK_BYTES = 16 * 1024 * 1024 EPHEMERAL_ROOTS = (Path("/dev/shm"), Path("/tmp")) EPHEMERAL_MARKER_NAME = ".psi0_source_split_ram_stage.json" _IGNORED_NAMES = { MANIFEST_NAME, "preflight_audit_report.json", ".artifact_stage_complete.json", EPHEMERAL_MARKER_NAME, } def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb", buffering=COPY_CHUNK_BYTES) as file: while chunk := file.read(COPY_CHUNK_BYTES): digest.update(chunk) return digest.hexdigest() def manifest_digest(manifest_path: Path) -> str: return hashlib.sha256(manifest_path.read_bytes()).hexdigest() def _safe_relative_path(value: str) -> Path: pure = PurePosixPath(str(value)) if pure.is_absolute() or not pure.parts or ".." in pure.parts: raise ValueError(f"Unsafe artifact path: {value!r}") return Path(*pure.parts) def _iter_artifact_files(root: Path): for path in sorted(root.rglob("*")): if not path.is_file(): continue relative = path.relative_to(root) if path.name in _IGNORED_NAMES: continue if any(part.startswith(".artifact_stage") for part in relative.parts): continue if path.name.endswith((".tmp", ".lock")): raise RuntimeError(f"Refusing to package temporary artifact: {path}") yield path, relative def _atomic_json(path: Path, value: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") try: with temporary.open("w", encoding="utf-8") as file: json.dump(value, file, indent=2, sort_keys=True) file.write("\n") file.flush() os.fsync(file.fileno()) os.replace(temporary, path) finally: try: temporary.unlink() except FileNotFoundError: pass def build_manifest(root: Path) -> Path: root = root.resolve() if not root.is_dir(): raise FileNotFoundError(f"Artifact root does not exist: {root}") started = time.perf_counter() files: list[dict[str, Any]] = [] total_bytes = 0 for index, (path, relative) in enumerate(_iter_artifact_files(root), start=1): stat = path.stat() size = int(stat.st_size) files.append( { "path": relative.as_posix(), "size": size, "mtime_ns": int(stat.st_mtime_ns), "sha256": sha256_file(path), } ) total_bytes += size print( f"[manifest] file={index} path={relative} size={size / 1024**3:.3f}GiB", flush=True, ) if not files: raise RuntimeError(f"No artifacts found under {root}") manifest = { "version": MANIFEST_VERSION, "source_split_version": 1, "files": files, "file_count": len(files), "total_bytes": total_bytes, } manifest_path = root / MANIFEST_NAME _atomic_json(manifest_path, manifest) elapsed = time.perf_counter() - started print( f"[manifest ready] files={len(files)} size={total_bytes / 1024**3:.3f}GiB " f"elapsed={elapsed:.1f}s sha256={manifest_digest(manifest_path)} path={manifest_path}", flush=True, ) return manifest_path def load_manifest(root: Path) -> tuple[Path, dict[str, Any]]: manifest_path = root / MANIFEST_NAME if not manifest_path.is_file(): raise FileNotFoundError(f"Artifact manifest does not exist: {manifest_path}") manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if int(manifest.get("version", 0)) != MANIFEST_VERSION: raise RuntimeError(f"Unsupported artifact manifest version: {manifest_path}") files = manifest.get("files") if not isinstance(files, list) or len(files) != int(manifest.get("file_count", -1)): raise RuntimeError(f"Malformed artifact manifest: {manifest_path}") paths = [str(entry.get("path", "")) for entry in files] if len(set(paths)) != len(paths): raise RuntimeError(f"Duplicate paths in artifact manifest: {manifest_path}") for value in paths: _safe_relative_path(value) expected_total = sum(int(entry["size"]) for entry in files) if expected_total != int(manifest.get("total_bytes", -1)): raise RuntimeError(f"Artifact manifest byte count mismatch: {manifest_path}") return manifest_path, manifest def verify_artifacts(root: Path) -> None: root = root.resolve() manifest_path, manifest = load_manifest(root) started = time.perf_counter() checked_bytes = 0 for index, entry in enumerate(manifest["files"], start=1): relative = _safe_relative_path(entry["path"]) path = root / relative expected_size = int(entry["size"]) if not path.is_file() or path.stat().st_size != expected_size: raise RuntimeError(f"Missing or truncated artifact: {path}") actual_sha256 = sha256_file(path) if actual_sha256 != str(entry["sha256"]): raise RuntimeError(f"SHA256 mismatch for artifact: {path}") checked_bytes += expected_size print(f"[verify] file={index}/{len(manifest['files'])} path={relative}", flush=True) elapsed = time.perf_counter() - started print( f"[verify complete] files={len(manifest['files'])} " f"size={checked_bytes / 1024**3:.3f}GiB elapsed={elapsed:.1f}s " f"rate={checked_bytes / 1024**3 / max(elapsed, 1e-9):.3f}GiB/s " f"manifest_sha256={manifest_digest(manifest_path)}", flush=True, ) class StageLock: def __init__(self, destination: Path, stale_seconds: int = 6 * 3600) -> None: self.path = destination / ".artifact_stage.lock" self.stale_seconds = int(stale_seconds) self.acquired = False def __enter__(self): last_message = 0.0 while True: try: self.path.mkdir() self.acquired = True return self except FileExistsError: try: age = time.time() - self.path.stat().st_mtime if age > self.stale_seconds: self.path.rmdir() continue except FileNotFoundError: continue now = time.time() if now - last_message >= 30.0: print(f"[stage] waiting for lock: {self.path}", flush=True) last_message = now time.sleep(1.0) def __exit__(self, exc_type, exc, traceback): if self.acquired: try: self.path.rmdir() except FileNotFoundError: pass def _copy_verified(source: Path, destination: Path, entry: dict[str, Any]) -> int: expected_size = int(entry["size"]) expected_sha256 = str(entry["sha256"]) if not source.is_file() or source.stat().st_size != expected_size: raise RuntimeError(f"Missing or truncated source artifact: {source}") destination.parent.mkdir(parents=True, exist_ok=True) temporary = destination.with_name(f".{destination.name}.{os.getpid()}.tmp") digest = hashlib.sha256() copied = 0 try: with source.open("rb", buffering=COPY_CHUNK_BYTES) as source_file, temporary.open( "wb", buffering=COPY_CHUNK_BYTES ) as destination_file: while chunk := source_file.read(COPY_CHUNK_BYTES): destination_file.write(chunk) digest.update(chunk) copied += len(chunk) destination_file.flush() os.fsync(destination_file.fileno()) if copied != expected_size or digest.hexdigest() != expected_sha256: raise RuntimeError(f"Source artifact failed checksum during copy: {source}") os.replace(temporary, destination) os.utime(destination, ns=(int(entry["mtime_ns"]), int(entry["mtime_ns"]))) finally: try: temporary.unlink() except FileNotFoundError: pass return copied def _require_ephemeral_destination(destination_root: Path) -> Path: destination_root = destination_root.resolve() for configured_root in EPHEMERAL_ROOTS: ephemeral_root = configured_root.resolve() try: relative = destination_root.relative_to(ephemeral_root) except ValueError: continue if not relative.parts: raise ValueError(f"Refusing to use the ephemeral filesystem root: {destination_root}") return destination_root roots = ", ".join(str(path.resolve()) for path in EPHEMERAL_ROOTS) raise ValueError( f"Ephemeral artifact destination must be under one of [{roots}]: {destination_root}" ) def stage_artifacts( source_root: Path, destination_root: Path, *, ephemeral: bool = False, ) -> None: source_root = source_root.resolve() destination_root = destination_root.resolve() if ephemeral: destination_root = _require_ephemeral_destination(destination_root) source_manifest_path, manifest = load_manifest(source_root) destination_root.mkdir(parents=True, exist_ok=True) started = time.perf_counter() copied_files = copied_bytes = reused_files = 0 with StageLock(destination_root): if ephemeral: free_bytes = int(shutil.disk_usage(destination_root).free) payload_bytes = int(manifest["total_bytes"]) required_bytes = payload_bytes + 1024**3 _atomic_json( destination_root / EPHEMERAL_MARKER_NAME, { "version": 1, "destination": str(destination_root), "source": str(source_root), "pid": os.getpid(), }, ) if free_bytes < required_bytes: raise RuntimeError( f"Insufficient ephemeral space for source-split artifacts: " f"payload={payload_bytes / 1024**3:.3f}GiB " f"required_with_margin={required_bytes / 1024**3:.3f}GiB " f"free={free_bytes / 1024**3:.3f}GiB destination={destination_root}" ) print( f"[ephemeral stage] payload={payload_bytes / 1024**3:.3f}GiB " f"required_with_margin={required_bytes / 1024**3:.3f}GiB " f"free={free_bytes / 1024**3:.3f}GiB destination={destination_root}", flush=True, ) for entry in manifest["files"]: relative = _safe_relative_path(entry["path"]) source = source_root / relative destination = destination_root / relative expected_size = int(entry["size"]) expected_mtime_ns = int(entry["mtime_ns"]) reusable = ( destination.is_file() and destination.stat().st_size == expected_size and destination.stat().st_mtime_ns == expected_mtime_ns ) if not reusable and destination.is_file() and destination.stat().st_size == expected_size: if sha256_file(destination) == str(entry["sha256"]): os.utime(destination, ns=(expected_mtime_ns, expected_mtime_ns)) reusable = True if reusable: reused_files += 1 continue copied_bytes += _copy_verified(source, destination, entry) copied_files += 1 print(f"[stage copied] {relative} ({expected_size / 1024**3:.3f}GiB)", flush=True) destination_manifest = destination_root / MANIFEST_NAME if source_manifest_path.resolve() != destination_manifest.resolve(): temporary_manifest = destination_manifest.with_name( f".{destination_manifest.name}.{os.getpid()}.tmp" ) try: shutil.copyfile(source_manifest_path, temporary_manifest) os.replace(temporary_manifest, destination_manifest) finally: try: temporary_manifest.unlink() except FileNotFoundError: pass _atomic_json( destination_root / ".artifact_stage_complete.json", { "version": 1, "manifest_sha256": manifest_digest(source_manifest_path), "file_count": len(manifest["files"]), "total_bytes": int(manifest["total_bytes"]), }, ) elapsed = time.perf_counter() - started print( f"[stage ready] source={source_root} destination={destination_root} " f"files={len(manifest['files'])} reused={reused_files} copied={copied_files} " f"copied_gib={copied_bytes / 1024**3:.3f} elapsed={elapsed:.1f}s", flush=True, ) def cleanup_ephemeral_artifacts(root: Path) -> None: root = root.resolve() if not root.exists(): print(f"[ephemeral cleanup] already absent: {root}", flush=True) return root = _require_ephemeral_destination(root) marker_path = root / EPHEMERAL_MARKER_NAME if not marker_path.is_file(): raise RuntimeError(f"Refusing to remove unowned ephemeral directory: {root}") marker = json.loads(marker_path.read_text(encoding="utf-8")) if int(marker.get("version", 0)) != 1 or str(marker.get("destination", "")) != str(root): raise RuntimeError(f"Invalid ephemeral ownership marker: {marker_path}") started = time.perf_counter() shutil.rmtree(root) print( f"[ephemeral cleanup] removed={root} elapsed={time.perf_counter() - started:.1f}s", flush=True, ) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Build, verify, or stage source-split artifacts.") subparsers = parser.add_subparsers(dest="command", required=True) build = subparsers.add_parser("build-manifest") build.add_argument("--root", type=Path, required=True) verify = subparsers.add_parser("verify") verify.add_argument("--root", type=Path, required=True) stage = subparsers.add_parser("stage") stage.add_argument("--source", type=Path, required=True) stage.add_argument("--destination", type=Path, required=True) stage.add_argument("--ephemeral", action="store_true") cleanup = subparsers.add_parser("cleanup-ephemeral") cleanup.add_argument("--root", type=Path, required=True) return parser.parse_args() def main() -> None: args = parse_args() if args.command == "build-manifest": build_manifest(args.root) elif args.command == "verify": verify_artifacts(args.root) elif args.command == "stage": stage_artifacts(args.source, args.destination, ephemeral=args.ephemeral) elif args.command == "cleanup-ephemeral": cleanup_ephemeral_artifacts(args.root) else: raise RuntimeError(f"Unhandled command: {args.command}") if __name__ == "__main__": main()