scripts/upload_single_file.py

import sys
from pathlib import Path

# Add parent directory to path to locate 'config' module
sys.path.append(str(Path(__file__).parent.parent))

from config.settings import Settings

import os
from huggingface_hub import HfApi

def verify_space_access():
    """Verify access to the Space and its settings"""
    api = HfApi(token=Settings.HF_TOKEN)
    
    try:
        # Check if we have write access
        space_info = api.space_info(Settings.HF_SPACE)
        print(f"Space runtime: {space_info.runtime}")
        print(f"Space sdk: {space_info.sdk}")
        
        # Try to list the contents to verify write access
        contents = api.list_repo_files(
            repo_id=Settings.HF_SPACE,
            repo_type="space"
        )
        print(f"Space contents: {len(contents)} files")
        return True
    except Exception as e:
        print(f"Error verifying space access: {str(e)}")
        return False

def main():
    """Upload a specific large embedding file to HuggingFace Space using HfApi.upload_file"""
    if not Settings.HF_TOKEN:
        raise ValueError("HF_TOKEN not found in environment variables")
        
    print("Verifying Space access...")
    if not verify_space_access():
        raise RuntimeError("Failed to verify Space access")
    
    print("Starting upload process...")
    
    # Define the file to upload
    local_file_path = Path(
        "./data/processed/embeddings/chroma/fade0013-ed4b-4928-b81b-7435145156dc/data_level0.bin"
    )
    if not local_file_path.exists():
        raise FileNotFoundError(f"File {local_file_path} does not exist")
    
    # Define the path in the repository
    repo_file_path = "data/processed/embeddings/chroma/fade0013-ed4b-4928-b81b-7435145156dc/data_level0.bin"
    
    api = HfApi(token=Settings.HF_TOKEN)
    
    print(f"Uploading {local_file_path} to {Settings.HF_SPACE}/{repo_file_path}...")
    try:
        api.upload_file(
            path_or_fileobj=str(local_file_path),
            path_in_repo=repo_file_path,
            repo_id=Settings.HF_SPACE,
            repo_type="space",
            token=Settings.HF_TOKEN,
        )
        print("Upload complete!")
    except Exception as e:
        print(f"Error during upload: {str(e)}")
        if "storage" in str(e).lower():
            print("\nNOTE: This might be a Git LFS limitation.")
            print("Consider splitting the file into smaller chunks or contacting Hugging Face support.")
        raise

if __name__ == "__main__":
    main()