code-HFD_lzk / scripts /data /manage_source_split_artifacts.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
8562e9b verified
Raw History Blame Contribute Delete
15.4 kB
from __future__ import annotations
import argparse
import hashlib
import json
import os
import shutil
import time
from pathlib import Path, PurePosixPath
from typing import Any
MANIFEST_NAME = "artifact_manifest.json"
MANIFEST_VERSION = 1
COPY_CHUNK_BYTES = 16 * 1024 * 1024
EPHEMERAL_ROOTS = (Path("/dev/shm"), Path("/tmp"))
EPHEMERAL_MARKER_NAME = ".psi0_source_split_ram_stage.json"
_IGNORED_NAMES = {
MANIFEST_NAME,
"preflight_audit_report.json",
".artifact_stage_complete.json",
EPHEMERAL_MARKER_NAME,
}
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb", buffering=COPY_CHUNK_BYTES) as file:
while chunk := file.read(COPY_CHUNK_BYTES):
digest.update(chunk)
return digest.hexdigest()
def manifest_digest(manifest_path: Path) -> str:
return hashlib.sha256(manifest_path.read_bytes()).hexdigest()
def _safe_relative_path(value: str) -> Path:
pure = PurePosixPath(str(value))
if pure.is_absolute() or not pure.parts or ".." in pure.parts:
raise ValueError(f"Unsafe artifact path: {value!r}")
return Path(*pure.parts)
def _iter_artifact_files(root: Path):
for path in sorted(root.rglob("*")):
if not path.is_file():
continue
relative = path.relative_to(root)
if path.name in _IGNORED_NAMES:
continue
if any(part.startswith(".artifact_stage") for part in relative.parts):
continue
if path.name.endswith((".tmp", ".lock")):
raise RuntimeError(f"Refusing to package temporary artifact: {path}")
yield path, relative
def _atomic_json(path: Path, value: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
try:
with temporary.open("w", encoding="utf-8") as file:
json.dump(value, file, indent=2, sort_keys=True)
file.write("\n")
file.flush()
os.fsync(file.fileno())
os.replace(temporary, path)
finally:
try:
temporary.unlink()
except FileNotFoundError:
pass
def build_manifest(root: Path) -> Path:
root = root.resolve()
if not root.is_dir():
raise FileNotFoundError(f"Artifact root does not exist: {root}")
started = time.perf_counter()
files: list[dict[str, Any]] = []
total_bytes = 0
for index, (path, relative) in enumerate(_iter_artifact_files(root), start=1):
stat = path.stat()
size = int(stat.st_size)
files.append(
{
"path": relative.as_posix(),
"size": size,
"mtime_ns": int(stat.st_mtime_ns),
"sha256": sha256_file(path),
}
)
total_bytes += size
print(
f"[manifest] file={index} path={relative} size={size / 1024**3:.3f}GiB",
flush=True,
)
if not files:
raise RuntimeError(f"No artifacts found under {root}")
manifest = {
"version": MANIFEST_VERSION,
"source_split_version": 1,
"files": files,
"file_count": len(files),
"total_bytes": total_bytes,
}
manifest_path = root / MANIFEST_NAME
_atomic_json(manifest_path, manifest)
elapsed = time.perf_counter() - started
print(
f"[manifest ready] files={len(files)} size={total_bytes / 1024**3:.3f}GiB "
f"elapsed={elapsed:.1f}s sha256={manifest_digest(manifest_path)} path={manifest_path}",
flush=True,
)
return manifest_path
def load_manifest(root: Path) -> tuple[Path, dict[str, Any]]:
manifest_path = root / MANIFEST_NAME
if not manifest_path.is_file():
raise FileNotFoundError(f"Artifact manifest does not exist: {manifest_path}")
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
if int(manifest.get("version", 0)) != MANIFEST_VERSION:
raise RuntimeError(f"Unsupported artifact manifest version: {manifest_path}")
files = manifest.get("files")
if not isinstance(files, list) or len(files) != int(manifest.get("file_count", -1)):
raise RuntimeError(f"Malformed artifact manifest: {manifest_path}")
paths = [str(entry.get("path", "")) for entry in files]
if len(set(paths)) != len(paths):
raise RuntimeError(f"Duplicate paths in artifact manifest: {manifest_path}")
for value in paths:
_safe_relative_path(value)
expected_total = sum(int(entry["size"]) for entry in files)
if expected_total != int(manifest.get("total_bytes", -1)):
raise RuntimeError(f"Artifact manifest byte count mismatch: {manifest_path}")
return manifest_path, manifest
def verify_artifacts(root: Path) -> None:
root = root.resolve()
manifest_path, manifest = load_manifest(root)
started = time.perf_counter()
checked_bytes = 0
for index, entry in enumerate(manifest["files"], start=1):
relative = _safe_relative_path(entry["path"])
path = root / relative
expected_size = int(entry["size"])
if not path.is_file() or path.stat().st_size != expected_size:
raise RuntimeError(f"Missing or truncated artifact: {path}")
actual_sha256 = sha256_file(path)
if actual_sha256 != str(entry["sha256"]):
raise RuntimeError(f"SHA256 mismatch for artifact: {path}")
checked_bytes += expected_size
print(f"[verify] file={index}/{len(manifest['files'])} path={relative}", flush=True)
elapsed = time.perf_counter() - started
print(
f"[verify complete] files={len(manifest['files'])} "
f"size={checked_bytes / 1024**3:.3f}GiB elapsed={elapsed:.1f}s "
f"rate={checked_bytes / 1024**3 / max(elapsed, 1e-9):.3f}GiB/s "
f"manifest_sha256={manifest_digest(manifest_path)}",
flush=True,
)
class StageLock:
def __init__(self, destination: Path, stale_seconds: int = 6 * 3600) -> None:
self.path = destination / ".artifact_stage.lock"
self.stale_seconds = int(stale_seconds)
self.acquired = False
def __enter__(self):
last_message = 0.0
while True:
try:
self.path.mkdir()
self.acquired = True
return self
except FileExistsError:
try:
age = time.time() - self.path.stat().st_mtime
if age > self.stale_seconds:
self.path.rmdir()
continue
except FileNotFoundError:
continue
now = time.time()
if now - last_message >= 30.0:
print(f"[stage] waiting for lock: {self.path}", flush=True)
last_message = now
time.sleep(1.0)
def __exit__(self, exc_type, exc, traceback):
if self.acquired:
try:
self.path.rmdir()
except FileNotFoundError:
pass
def _copy_verified(source: Path, destination: Path, entry: dict[str, Any]) -> int:
expected_size = int(entry["size"])
expected_sha256 = str(entry["sha256"])
if not source.is_file() or source.stat().st_size != expected_size:
raise RuntimeError(f"Missing or truncated source artifact: {source}")
destination.parent.mkdir(parents=True, exist_ok=True)
temporary = destination.with_name(f".{destination.name}.{os.getpid()}.tmp")
digest = hashlib.sha256()
copied = 0
try:
with source.open("rb", buffering=COPY_CHUNK_BYTES) as source_file, temporary.open(
"wb", buffering=COPY_CHUNK_BYTES
) as destination_file:
while chunk := source_file.read(COPY_CHUNK_BYTES):
destination_file.write(chunk)
digest.update(chunk)
copied += len(chunk)
destination_file.flush()
os.fsync(destination_file.fileno())
if copied != expected_size or digest.hexdigest() != expected_sha256:
raise RuntimeError(f"Source artifact failed checksum during copy: {source}")
os.replace(temporary, destination)
os.utime(destination, ns=(int(entry["mtime_ns"]), int(entry["mtime_ns"])))
finally:
try:
temporary.unlink()
except FileNotFoundError:
pass
return copied
def _require_ephemeral_destination(destination_root: Path) -> Path:
destination_root = destination_root.resolve()
for configured_root in EPHEMERAL_ROOTS:
ephemeral_root = configured_root.resolve()
try:
relative = destination_root.relative_to(ephemeral_root)
except ValueError:
continue
if not relative.parts:
raise ValueError(f"Refusing to use the ephemeral filesystem root: {destination_root}")
return destination_root
roots = ", ".join(str(path.resolve()) for path in EPHEMERAL_ROOTS)
raise ValueError(
f"Ephemeral artifact destination must be under one of [{roots}]: {destination_root}"
)
def stage_artifacts(
source_root: Path,
destination_root: Path,
*,
ephemeral: bool = False,
) -> None:
source_root = source_root.resolve()
destination_root = destination_root.resolve()
if ephemeral:
destination_root = _require_ephemeral_destination(destination_root)
source_manifest_path, manifest = load_manifest(source_root)
destination_root.mkdir(parents=True, exist_ok=True)
started = time.perf_counter()
copied_files = copied_bytes = reused_files = 0
with StageLock(destination_root):
if ephemeral:
free_bytes = int(shutil.disk_usage(destination_root).free)
payload_bytes = int(manifest["total_bytes"])
required_bytes = payload_bytes + 1024**3
_atomic_json(
destination_root / EPHEMERAL_MARKER_NAME,
{
"version": 1,
"destination": str(destination_root),
"source": str(source_root),
"pid": os.getpid(),
},
)
if free_bytes < required_bytes:
raise RuntimeError(
f"Insufficient ephemeral space for source-split artifacts: "
f"payload={payload_bytes / 1024**3:.3f}GiB "
f"required_with_margin={required_bytes / 1024**3:.3f}GiB "
f"free={free_bytes / 1024**3:.3f}GiB destination={destination_root}"
)
print(
f"[ephemeral stage] payload={payload_bytes / 1024**3:.3f}GiB "
f"required_with_margin={required_bytes / 1024**3:.3f}GiB "
f"free={free_bytes / 1024**3:.3f}GiB destination={destination_root}",
flush=True,
)
for entry in manifest["files"]:
relative = _safe_relative_path(entry["path"])
source = source_root / relative
destination = destination_root / relative
expected_size = int(entry["size"])
expected_mtime_ns = int(entry["mtime_ns"])
reusable = (
destination.is_file()
and destination.stat().st_size == expected_size
and destination.stat().st_mtime_ns == expected_mtime_ns
)
if not reusable and destination.is_file() and destination.stat().st_size == expected_size:
if sha256_file(destination) == str(entry["sha256"]):
os.utime(destination, ns=(expected_mtime_ns, expected_mtime_ns))
reusable = True
if reusable:
reused_files += 1
continue
copied_bytes += _copy_verified(source, destination, entry)
copied_files += 1
print(f"[stage copied] {relative} ({expected_size / 1024**3:.3f}GiB)", flush=True)
destination_manifest = destination_root / MANIFEST_NAME
if source_manifest_path.resolve() != destination_manifest.resolve():
temporary_manifest = destination_manifest.with_name(
f".{destination_manifest.name}.{os.getpid()}.tmp"
)
try:
shutil.copyfile(source_manifest_path, temporary_manifest)
os.replace(temporary_manifest, destination_manifest)
finally:
try:
temporary_manifest.unlink()
except FileNotFoundError:
pass
_atomic_json(
destination_root / ".artifact_stage_complete.json",
{
"version": 1,
"manifest_sha256": manifest_digest(source_manifest_path),
"file_count": len(manifest["files"]),
"total_bytes": int(manifest["total_bytes"]),
},
)
elapsed = time.perf_counter() - started
print(
f"[stage ready] source={source_root} destination={destination_root} "
f"files={len(manifest['files'])} reused={reused_files} copied={copied_files} "
f"copied_gib={copied_bytes / 1024**3:.3f} elapsed={elapsed:.1f}s",
flush=True,
)
def cleanup_ephemeral_artifacts(root: Path) -> None:
root = root.resolve()
if not root.exists():
print(f"[ephemeral cleanup] already absent: {root}", flush=True)
return
root = _require_ephemeral_destination(root)
marker_path = root / EPHEMERAL_MARKER_NAME
if not marker_path.is_file():
raise RuntimeError(f"Refusing to remove unowned ephemeral directory: {root}")
marker = json.loads(marker_path.read_text(encoding="utf-8"))
if int(marker.get("version", 0)) != 1 or str(marker.get("destination", "")) != str(root):
raise RuntimeError(f"Invalid ephemeral ownership marker: {marker_path}")
started = time.perf_counter()
shutil.rmtree(root)
print(
f"[ephemeral cleanup] removed={root} elapsed={time.perf_counter() - started:.1f}s",
flush=True,
)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Build, verify, or stage source-split artifacts.")
subparsers = parser.add_subparsers(dest="command", required=True)
build = subparsers.add_parser("build-manifest")
build.add_argument("--root", type=Path, required=True)
verify = subparsers.add_parser("verify")
verify.add_argument("--root", type=Path, required=True)
stage = subparsers.add_parser("stage")
stage.add_argument("--source", type=Path, required=True)
stage.add_argument("--destination", type=Path, required=True)
stage.add_argument("--ephemeral", action="store_true")
cleanup = subparsers.add_parser("cleanup-ephemeral")
cleanup.add_argument("--root", type=Path, required=True)
return parser.parse_args()
def main() -> None:
args = parse_args()
if args.command == "build-manifest":
build_manifest(args.root)
elif args.command == "verify":
verify_artifacts(args.root)
elif args.command == "stage":
stage_artifacts(args.source, args.destination, ephemeral=args.ephemeral)
elif args.command == "cleanup-ephemeral":
cleanup_ephemeral_artifacts(args.root)
else:
raise RuntimeError(f"Unhandled command: {args.command}")
if __name__ == "__main__":
main()