paper-qa-rag / script /storage.py
whosouravsharma's picture
Upload folder using huggingface_hub
50b6391 verified
Raw History Blame Contribute Delete
842 Bytes
import json
import subprocess
import time
import uuid
LOGS_BUCKET = "hf://buckets/whosouravsharma/paper-qa-logs"
FILES_BUCKET = "hf://buckets/whosouravsharma/paper-qa-files"
def _cp_stdin(data: bytes, dest_uri: str) -> None:
subprocess.run(
["hf", "buckets", "cp", "-", dest_uri],
input=data,
check=True,
capture_output=True,
)
def log_event(event: str, **fields) -> None:
record = {"event": event, "ts": time.time(), **fields}
line = (json.dumps(record) + "\n").encode()
dest = f"{LOGS_BUCKET}/{time.strftime('%Y-%m-%d')}/{event}-{uuid.uuid4().hex}.jsonl"
try:
_cp_stdin(line, dest)
except Exception:
pass
def store_file(data: bytes, filename: str) -> str:
dest = f"{FILES_BUCKET}/{uuid.uuid4().hex}_{filename}"
_cp_stdin(data, dest)
return dest