"""Upload the tokenised data folder to the private HF dataset QLNI/embeddings-are-overpriced-data.

    python upload_data.py --data data
"""
import argparse
from huggingface_hub import HfApi

ap = argparse.ArgumentParser()
ap.add_argument("--data", default="data")
a = ap.parse_args()
api = HfApi()
api.create_repo("QLNI/embeddings-are-overpriced-data", repo_type="dataset", private=True, exist_ok=True)
api.upload_large_folder(repo_id="QLNI/embeddings-are-overpriced-data", repo_type="dataset", folder_path=a.data,
                        allow_patterns=["*.npy"])
print("uploaded", a.data, flush=True)