scripts/push_dataset_to_hf.py

"""Upload the competition dataset to the private Hugging Face repo.

Split out from the code/weights push because the 600 page-scale PNGs are 2.5 GB and take
several hours on a home uplink, while the code and checkpoints take under an hour. Run this
whenever the machine can stay awake and online — overnight is the intended use.

    .venv/bin/python scripts/push_dataset_to_hf.py

Resumable: `upload_large_folder` records per-file state in .cache/huggingface inside the folder,
so a killed run picks up where it stopped instead of re-uploading. Safe to run repeatedly.
"""
from huggingface_hub import HfApi

REPO = "siddhant20/task1"
ROOT = "/Users/siddhantparashar/projects/TASK1_CYBER"

if __name__ == "__main__":
    HfApi().upload_large_folder(
        repo_id=REPO,
        repo_type="model",
        folder_path=ROOT,
        # Everything except the dataset is already pushed; the zip is a duplicate of the
        # extracted folder and would double the transfer for nothing.
        allow_patterns=["Task1/**"],
        ignore_patterns=["*.zip", "**/.DS_Store"],
        num_workers=8,
        print_report_every=60,
    )
    print("dataset upload complete")