Buckets:
| """cwm3 run provenance — verbatim copy of cwm2/data/runmeta.py (2026-09-10): run.json writer (command, code hash, inputs sha256, seed, timestamps). | |
| Results directories are append-only: run.json is never overwritten; each run writes | |
| run_<utc>.json and (optionally) updates a `latest` pointer file. | |
| """ | |
| import hashlib | |
| import json | |
| import os | |
| import socket | |
| import sys | |
| import time | |
| from pathlib import Path | |
| def sha256_file(p, chunk=1 << 22): | |
| h = hashlib.sha256() | |
| with open(p, "rb") as f: | |
| for b in iter(lambda: f.read(chunk), b""): | |
| h.update(b) | |
| return h.hexdigest() | |
| def code_hash(files): | |
| """sha256 over the concatenated source of the given files (git-free).""" | |
| h = hashlib.sha256() | |
| for p in files: | |
| h.update(str(Path(p).name).encode()) | |
| h.update(Path(p).read_bytes()) | |
| return h.hexdigest() | |
| def inputs_digest(paths, listing_path=None): | |
| """Per-file sha256 (sorted); returns {n, digest} and optionally writes a listing.""" | |
| rows = [] | |
| for p in sorted(map(str, paths)): | |
| rows.append((p, sha256_file(p), os.path.getsize(p))) | |
| h = hashlib.sha256() | |
| for p, d, s in rows: | |
| h.update(f"{p}\t{d}\t{s}\n".encode()) | |
| if listing_path is not None: | |
| with open(listing_path, "a") as f: | |
| f.write(f"# listing written {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}\n") | |
| for p, d, s in rows: | |
| f.write(f"{d}\t{s}\t{p}\n") | |
| return {"n_files": len(rows), "digest": h.hexdigest(), | |
| "total_bytes": int(sum(s for _, _, s in rows))} | |
| def write_run_json(out_dir, name, code_files, inputs, seed, t_start, extra=None, | |
| input_listing=True): | |
| out_dir = Path(out_dir) | |
| out_dir.mkdir(parents=True, exist_ok=True) | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| listing = out_dir / f"inputs_sha256_{name}.txt" if input_listing else None | |
| rec = { | |
| "name": name, | |
| "command": " ".join([sys.executable] + sys.argv), | |
| "cwd": os.getcwd(), | |
| "host": socket.gethostname(), | |
| "python": sys.version.split()[0], | |
| "code_files": [str(p) for p in code_files], | |
| "code_sha256": code_hash(code_files), | |
| "inputs": inputs_digest(inputs, listing), | |
| "seed": seed, | |
| "t_start_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(t_start)), | |
| "t_end_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), | |
| "wall_s": round(time.time() - t_start, 1), | |
| } | |
| if extra: | |
| rec.update(extra) | |
| p = out_dir / f"run_{name}_{stamp}.json" | |
| if p.exists(): # 2026-09-11: concurrent evaluation shells can share the same UTC second -> unique microsecond suffix instead of an assert | |
| p = out_dir / f"run_{name}_{stamp}_{int((time.time() % 1) * 1e6):06d}.json" | |
| while p.exists(): | |
| p = out_dir / f"run_{name}_{stamp}_{int((time.time() % 1) * 1e6):06d}.json" | |
| p.write_text(json.dumps(rec, indent=1, default=str)) | |
| # convenience pointer (the timestamped file is the record of truth) | |
| (out_dir / f"run_{name}_latest.json").write_text(json.dumps(rec, indent=1, default=str)) | |
| return p | |
Xet Storage Details
- Size:
- 3.17 kB
- Xet hash:
- 560af2b5cc2cd479622424625377278e4de7e2989f571e2dfd7f8ae380c42950
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.