Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
| #!/usr/bin/env python3 | |
| """Upload a full example corpus to object storage (R2) and report its stats. | |
| Example corpora larger than a few hundred KB do NOT live in the git repo: they | |
| would ride in the Docker image and the Space repo on every deploy. They live in | |
| the configured bucket under `examples/`, a prefix nothing else writes to and | |
| retention never sweeps, so a user's copy can be deleted without touching the | |
| master. | |
| The row count and columns this prints are what belongs in the catalogue entry | |
| in backend/app/example_corpora.py; a test asserts the two agree. | |
| Usage: | |
| python scripts/upload_example_corpus.py CAMEL.csv camel_full.csv | |
| """ | |
| import os | |
| import sys | |
| from pathlib import Path | |
| REPO = Path(__file__).resolve().parents[1] | |
| def load_env() -> None: | |
| """Load .env the way the app does. Values are never printed.""" | |
| env = REPO / ".env" | |
| if not env.exists(): | |
| return | |
| for line in env.read_text().splitlines(): | |
| if line.strip() and not line.startswith("#") and "=" in line: | |
| key, value = line.split("=", 1) | |
| os.environ.setdefault(key.strip(), value.strip()) | |
| def main() -> None: | |
| if len(sys.argv) < 3: | |
| raise SystemExit(__doc__) | |
| src, dest_name = Path(sys.argv[1]), sys.argv[2] | |
| if not src.exists(): | |
| raise SystemExit(f"no such file: {src}") | |
| load_env() | |
| sys.path.insert(0, str(REPO / "backend")) | |
| import pandas as pd | |
| from app import storage | |
| if storage.backend() != "s3": | |
| raise SystemExit("CCR_STORAGE is not s3; nothing to upload to.") | |
| df = pd.read_csv(src, low_memory=False) | |
| key = f"examples/{dest_name}" | |
| storage._s3().upload_file(str(src), storage._bucket(), key) | |
| print(f"uploaded -> s3://{key} ({src.stat().st_size / 1e6:.1f} MB)") | |
| print(f"n_rows = {len(df)}") | |
| print(f"columns = {len(df.columns)}") | |
| print(f"text column = {'text' if 'text' in df.columns else '???'}") | |
| print("\nPut n_rows in the example_corpora.py entry for this corpus.") | |
| if __name__ == "__main__": | |
| main() | |