Download scripts/upload_data_to_hf.py from KLEB38/OC_P8: direct link, hf CLI and curl.
- Browser
- Download file 2.09 kB
-
https://huggingface.co/spaces/KLEB38/OC_P8/resolve/main/scripts/upload_data_to_hf.py
- Command line
-
hf download hf://spaces/KLEB38/OC_P8/scripts/upload_data_to_hf.py
-
curl -L -o upload_data_to_hf.py https://huggingface.co/spaces/KLEB38/OC_P8/resolve/main/scripts/upload_data_to_hf.py
2.09 kB
| """Upload the feature store parquet to the companion HF Dataset repo. | |
| HF Spaces silently drop files larger than 10 MB on direct upload. The | |
| recommended pattern is therefore to host large data files in a Dataset | |
| repo and have the Space download them at runtime via `hf_hub_download`. | |
| PREREQUISITE — create the Dataset once on https://huggingface.co/new-dataset | |
| Owner: KLEB38 | |
| Name: oc-p8-features | |
| Visibility: public (or private + HF_TOKEN secret on the Space) | |
| Usage (PowerShell): | |
| $env:HF_TOKEN = "hf_xxx..." | |
| uv run python scripts/upload_data_to_hf.py | |
| """ | |
| from __future__ import annotations | |
| import os | |
| from pathlib import Path | |
| from huggingface_hub import CommitOperationAdd, HfApi | |
| REPO_ID = "KLEB38/oc-p8-features" | |
| REPO_TYPE = "dataset" | |
| PARQUET_PATH = Path("data/features_store.parquet") | |
| def main() -> None: | |
| token = os.environ.get("HF_TOKEN") | |
| if not token: | |
| raise SystemExit("HF_TOKEN env var is required") | |
| if not PARQUET_PATH.exists(): | |
| raise SystemExit( | |
| f"{PARQUET_PATH} not found - run " | |
| "`uv run python scripts/build_feature_store.py` first." | |
| ) | |
| size_mb = PARQUET_PATH.stat().st_size / (1024 * 1024) | |
| print(f"Local file: {PARQUET_PATH} ({size_mb:.1f} MB)") | |
| api = HfApi(token=token) | |
| print(f"Authenticated as: {api.whoami()['name']}") | |
| print(f"\nUploading to {REPO_TYPE} repo: {REPO_ID}") | |
| commit_info = api.create_commit( | |
| repo_id=REPO_ID, | |
| repo_type=REPO_TYPE, | |
| operations=[ | |
| CommitOperationAdd( | |
| path_in_repo="features_store.parquet", | |
| path_or_fileobj=str(PARQUET_PATH), | |
| ), | |
| ], | |
| commit_message="Update features store parquet", | |
| ) | |
| print(f"Commit OID: {commit_info.oid}") | |
| print(f"Commit URL: {commit_info.commit_url}") | |
| print("\nFiles AFTER upload:") | |
| for f in api.list_repo_files(repo_id=REPO_ID, repo_type=REPO_TYPE): | |
| marker = " <-- TARGET" if f == "features_store.parquet" else "" | |
| print(f" - {f}{marker}") | |
| if __name__ == "__main__": | |
| main() | |