Spaces:
Running
Running
File size: 4,039 Bytes
e317359 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 | #!/usr/bin/env python3
"""Build a portable seed from public source responses and metrics, without secrets."""
from datetime import datetime, timezone
import gzip
import hashlib
import io
import json
from pathlib import Path
import tarfile
import subprocess
import sys
import tempfile
ROOT = Path(__file__).resolve().parents[1]
def main():
source = ROOT / "space/vendor/ts_bench/data/sample_week"
files = [p for name in ("normalized", "raw")
for p in (source / name).rglob("*") if p.is_file()]
if (source / "summary.json").exists():
files.append(source / "summary.json")
for p in (ROOT / "space/results").rglob("*"):
if not p.is_file():
continue
rel = p.relative_to(ROOT / "space/results")
if any(part in rel.parts for part in ("prequential", "saved_evals", "forecasts", "canonical_all_history")):
continue
if p.suffix not in (".json", ".csv", ".tex", ".jsonl") or p.name == "eval_state.json":
continue
if p.suffix == ".jsonl" and rel.as_posix() not in ("evaluation_metrics_canonical.jsonl", "eval_history.jsonl"):
continue
files.append(p)
# Check literal configured credentials before any file enters the archive.
from dotenv import dotenv_values
secrets = [v.encode() for k, v in dotenv_values(ROOT / ".env").items()
if v and len(v) >= 12 and any(word in k for word in ("TOKEN", "KEY", "SECRET", "PASSWORD"))]
members = []
for p in sorted(files):
content = p.read_bytes()
if any(secret in content for secret in secrets):
raise SystemExit(f"Credential found in seed candidate: {p.relative_to(ROOT)}")
if p.name == "online_status.json":
status = json.loads(content)
status.update(status="bootstrap", issued_forecasts=0, resolved_forecasts=0,
pending_tasks=0, issued_tasks=0, push_status="not_published",
bootstrap_note="Private pending forecast state is excluded; start a new live cycle.")
content = (json.dumps(status, indent=2) + "\n").encode()
members.append((p.relative_to(ROOT).as_posix(), content))
# Export into an empty directory: old runtime exports otherwise accumulate
# tens of thousands of unreachable task files across collection cycles.
with tempfile.TemporaryDirectory(prefix="livehouse-seed-inputs-") as temp:
subprocess.run([sys.executable, str(ROOT / "space/vendor/ts_bench/export_model_inputs.py"),
"--input", str(source / "normalized"), "--output", temp, "--limit", "20"], check=True)
prefix = "space/vendor/ts_bench/data/sample_week/model_inputs_latest/"
members.extend((prefix + p.relative_to(temp).as_posix(), p.read_bytes())
for p in Path(temp).rglob("*") if p.is_file())
archive = ROOT / "bootstrap/seed.tar.gz"
with archive.open("wb") as raw, gzip.GzipFile(filename="", fileobj=raw, mode="wb", mtime=0) as compressed:
with tarfile.open(fileobj=compressed, mode="w") as bundle:
for name, content in members:
info = tarfile.TarInfo(name)
info.size = len(content)
info.mode = 0o644
bundle.addfile(info, io.BytesIO(content))
manifest = {"snapshot_date": datetime.now(timezone.utc).date().isoformat(),
"sha256": hashlib.sha256(archive.read_bytes()).hexdigest(),
"bytes": archive.stat().st_size, "files": len(members),
"contents": "Current public raw responses, normalized observations, latest inputs, historical canonical metrics, public tables and forecast visualization snapshots. No credentials or private task/inference archives.",
"historical_raw_available": False, "current_raw_included": True}
(ROOT / "bootstrap/manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
print(json.dumps(manifest, indent=2))
if __name__ == "__main__":
main()
|