load_app_data.py

import os
from dotenv import load_dotenv
from huggingface_hub import snapshot_download

load_dotenv("/app/config.env")

HF_TOKEN = os.environ["HF_TOKEN"]
APP_DATA_REPO = os.environ["APP_DATA_REPO"]
APP_DATA_REPO_BRANCH = os.environ["APP_DATA_REPO_BRANCH"]

snapshot_download(
    repo_id=f"{APP_DATA_REPO}",
    revision=APP_DATA_REPO_BRANCH,
    repo_type="dataset",
    local_dir="/app/potato",
    token=HF_TOKEN
)

print(f"downloaded app data from {APP_DATA_REPO} to /app/potato")