260331_mo5_format_diverse / scripts /pipeline_utils.py
jprivera's picture
upload batch 1/2 from pod 38.80.152.76
1878182 verified
Raw History Blame Contribute Delete
3.08 kB
"""Shared utilities for MO4 scheming pipeline reproducibility."""
import hashlib
import json
import subprocess
import sys
import uuid
from datetime import datetime, timezone
from pathlib import Path
def make_record_id() -> str:
"""Generate a unique record ID."""
return uuid.uuid4().hex[:12]
def get_git_hash() -> str:
"""Get current git commit hash, or 'unknown' if not in a repo."""
try:
result = subprocess.run(
["git", "rev-parse", "HEAD"],
capture_output=True, text=True, timeout=5,
)
if result.returncode == 0:
return result.stdout.strip()
except Exception:
pass
return "unknown"
def file_sha256(path: str | Path) -> str:
"""Compute SHA-256 hash of a file."""
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(8192), b""):
h.update(chunk)
return h.hexdigest()
def write_manifest(
output_path: str | Path,
*,
script: str,
args: dict,
input_files: dict[str, str | Path] | None = None,
output_files: dict[str, str | Path] | None = None,
counts: dict[str, int] | None = None,
extra: dict | None = None,
) -> Path:
"""Write a run manifest JSON file next to the output.
Args:
output_path: Primary output file path (manifest written as {stem}_manifest.json)
script: Script name (e.g. "01_generate_ideas.py")
args: CLI arguments as dict
input_files: {label: path} — hashes computed automatically
output_files: {label: path} — hashes computed automatically
counts: {label: count} — record counts
extra: Any additional metadata
"""
output_path = Path(output_path)
manifest_path = output_path.parent / f"{output_path.stem}_manifest.json"
manifest = {
"script": script,
"timestamp": datetime.now(timezone.utc).isoformat(),
"git_hash": get_git_hash(),
"python_version": sys.version,
"cli_args": args,
}
if input_files:
manifest["input_files"] = {}
for label, path in input_files.items():
p = Path(path)
if p.exists():
manifest["input_files"][label] = {
"path": str(p),
"sha256": file_sha256(p),
}
else:
manifest["input_files"][label] = {
"path": str(p),
"sha256": "FILE_NOT_FOUND",
}
if output_files:
manifest["output_files"] = {}
for label, path in output_files.items():
p = Path(path)
if p.exists():
manifest["output_files"][label] = {
"path": str(p),
"sha256": file_sha256(p),
}
if counts:
manifest["counts"] = counts
if extra:
manifest["extra"] = extra
with open(manifest_path, "w") as f:
json.dump(manifest, f, indent=2)
print(f" Manifest: {manifest_path}")
return manifest_path