Download scripts/pipeline_utils.py from jprivera/260331_mo5_format_diverse: direct link, hf CLI and curl.
- Browser
- Download file 3.08 kB
-
https://huggingface.co/jprivera/260331_mo5_format_diverse/resolve/main/scripts/pipeline_utils.py
- Command line
-
hf download hf://jprivera/260331_mo5_format_diverse/scripts/pipeline_utils.py
-
curl -L -o pipeline_utils.py https://huggingface.co/jprivera/260331_mo5_format_diverse/resolve/main/scripts/pipeline_utils.py
3.08 kB
| """Shared utilities for MO4 scheming pipeline reproducibility.""" | |
| import hashlib | |
| import json | |
| import subprocess | |
| import sys | |
| import uuid | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| def make_record_id() -> str: | |
| """Generate a unique record ID.""" | |
| return uuid.uuid4().hex[:12] | |
| def get_git_hash() -> str: | |
| """Get current git commit hash, or 'unknown' if not in a repo.""" | |
| try: | |
| result = subprocess.run( | |
| ["git", "rev-parse", "HEAD"], | |
| capture_output=True, text=True, timeout=5, | |
| ) | |
| if result.returncode == 0: | |
| return result.stdout.strip() | |
| except Exception: | |
| pass | |
| return "unknown" | |
| def file_sha256(path: str | Path) -> str: | |
| """Compute SHA-256 hash of a file.""" | |
| h = hashlib.sha256() | |
| with open(path, "rb") as f: | |
| for chunk in iter(lambda: f.read(8192), b""): | |
| h.update(chunk) | |
| return h.hexdigest() | |
| def write_manifest( | |
| output_path: str | Path, | |
| *, | |
| script: str, | |
| args: dict, | |
| input_files: dict[str, str | Path] | None = None, | |
| output_files: dict[str, str | Path] | None = None, | |
| counts: dict[str, int] | None = None, | |
| extra: dict | None = None, | |
| ) -> Path: | |
| """Write a run manifest JSON file next to the output. | |
| Args: | |
| output_path: Primary output file path (manifest written as {stem}_manifest.json) | |
| script: Script name (e.g. "01_generate_ideas.py") | |
| args: CLI arguments as dict | |
| input_files: {label: path} — hashes computed automatically | |
| output_files: {label: path} — hashes computed automatically | |
| counts: {label: count} — record counts | |
| extra: Any additional metadata | |
| """ | |
| output_path = Path(output_path) | |
| manifest_path = output_path.parent / f"{output_path.stem}_manifest.json" | |
| manifest = { | |
| "script": script, | |
| "timestamp": datetime.now(timezone.utc).isoformat(), | |
| "git_hash": get_git_hash(), | |
| "python_version": sys.version, | |
| "cli_args": args, | |
| } | |
| if input_files: | |
| manifest["input_files"] = {} | |
| for label, path in input_files.items(): | |
| p = Path(path) | |
| if p.exists(): | |
| manifest["input_files"][label] = { | |
| "path": str(p), | |
| "sha256": file_sha256(p), | |
| } | |
| else: | |
| manifest["input_files"][label] = { | |
| "path": str(p), | |
| "sha256": "FILE_NOT_FOUND", | |
| } | |
| if output_files: | |
| manifest["output_files"] = {} | |
| for label, path in output_files.items(): | |
| p = Path(path) | |
| if p.exists(): | |
| manifest["output_files"][label] = { | |
| "path": str(p), | |
| "sha256": file_sha256(p), | |
| } | |
| if counts: | |
| manifest["counts"] = counts | |
| if extra: | |
| manifest["extra"] = extra | |
| with open(manifest_path, "w") as f: | |
| json.dump(manifest, f, indent=2) | |
| print(f" Manifest: {manifest_path}") | |
| return manifest_path | |