scripts/upload_example_corpus.py

#!/usr/bin/env python3
"""Upload a full example corpus to object storage (R2) and report its stats.

Example corpora larger than a few hundred KB do NOT live in the git repo: they
would ride in the Docker image and the Space repo on every deploy. They live in
the configured bucket under `examples/`, a prefix nothing else writes to and
retention never sweeps, so a user's copy can be deleted without touching the
master.

The row count and columns this prints are what belongs in the catalogue entry
in backend/app/example_corpora.py; a test asserts the two agree.

Usage:
    python scripts/upload_example_corpus.py CAMEL.csv camel_full.csv
"""

import os
import sys
from pathlib import Path

REPO = Path(__file__).resolve().parents[1]


def load_env() -> None:
    """Load .env the way the app does. Values are never printed."""
    env = REPO / ".env"
    if not env.exists():
        return
    for line in env.read_text().splitlines():
        if line.strip() and not line.startswith("#") and "=" in line:
            key, value = line.split("=", 1)
            os.environ.setdefault(key.strip(), value.strip())


def main() -> None:
    if len(sys.argv) < 3:
        raise SystemExit(__doc__)
    src, dest_name = Path(sys.argv[1]), sys.argv[2]
    if not src.exists():
        raise SystemExit(f"no such file: {src}")

    load_env()
    sys.path.insert(0, str(REPO / "backend"))
    import pandas as pd

    from app import storage

    if storage.backend() != "s3":
        raise SystemExit("CCR_STORAGE is not s3; nothing to upload to.")

    df = pd.read_csv(src, low_memory=False)
    key = f"examples/{dest_name}"
    storage._s3().upload_file(str(src), storage._bucket(), key)

    print(f"uploaded -> s3://{key}  ({src.stat().st_size / 1e6:.1f} MB)")
    print(f"n_rows      = {len(df)}")
    print(f"columns     = {len(df.columns)}")
    print(f"text column = {'text' if 'text' in df.columns else '???'}")
    print("\nPut n_rows in the example_corpora.py entry for this corpus.")


if __name__ == "__main__":
    main()