Blame for script/storage.py

import json
import subprocess
import time
import uuid

LOGS_BUCKET = "hf://buckets/whosouravsharma/paper-qa-logs"
FILES_BUCKET = "hf://buckets/whosouravsharma/paper-qa-files"


def _cp_stdin(data: bytes, dest_uri: str) -> None:
    subprocess.run(
        ["hf", "buckets", "cp", "-", dest_uri],
        input=data,
        check=True,
        capture_output=True,
    )


def log_event(event: str, **fields) -> None:
    record = {"event": event, "ts": time.time(), **fields}
    line = (json.dumps(record) + "\n").encode()
    dest = f"{LOGS_BUCKET}/{time.strftime('%Y-%m-%d')}/{event}-{uuid.uuid4().hex}.jsonl"
    try:
        _cp_stdin(line, dest)
    except Exception:
        pass


def store_file(data: bytes, filename: str) -> str:
    dest = f"{FILES_BUCKET}/{uuid.uuid4().hex}_{filename}"
    _cp_stdin(data, dest)
    return dest