James Edmunds commited on
Commit ·
fecd36c
1
Parent(s): 3913938
feat: add upload verification for large files
Browse files- scripts/upload_embeddings.py +35 -4
scripts/upload_embeddings.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
| 1 |
"""Upload embeddings to HuggingFace Space"""
|
| 2 |
import sys
|
| 3 |
import os
|
|
|
|
| 4 |
from pathlib import Path
|
| 5 |
from huggingface_hub import HfApi, CommitOperationAdd
|
| 6 |
from dotenv import load_dotenv
|
|
@@ -32,9 +33,32 @@ def verify_space_access():
|
|
| 32 |
return False
|
| 33 |
|
| 34 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
def upload_large_file(api: HfApi, file_path: Path, repo_path: str):
|
| 36 |
"""Upload a single large file to the Space"""
|
| 37 |
-
|
|
|
|
| 38 |
|
| 39 |
# Create the commit operation
|
| 40 |
operation = CommitOperationAdd(
|
|
@@ -47,10 +71,17 @@ def upload_large_file(api: HfApi, file_path: Path, repo_path: str):
|
|
| 47 |
repo_id=Settings.HF_SPACE,
|
| 48 |
repo_type="space",
|
| 49 |
operations=[operation],
|
| 50 |
-
commit_message=f"Upload {file_path.name}"
|
| 51 |
-
run_as_future=True # Handle large files asynchronously
|
| 52 |
)
|
| 53 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
|
| 55 |
|
| 56 |
def main():
|
|
|
|
| 1 |
"""Upload embeddings to HuggingFace Space"""
|
| 2 |
import sys
|
| 3 |
import os
|
| 4 |
+
import time
|
| 5 |
from pathlib import Path
|
| 6 |
from huggingface_hub import HfApi, CommitOperationAdd
|
| 7 |
from dotenv import load_dotenv
|
|
|
|
| 33 |
return False
|
| 34 |
|
| 35 |
|
| 36 |
+
def wait_for_file_upload(api: HfApi, file_path: Path, repo_path: str):
|
| 37 |
+
"""Wait for a file to appear in the repository"""
|
| 38 |
+
max_attempts = 30 # 5 minutes total
|
| 39 |
+
for attempt in range(max_attempts):
|
| 40 |
+
try:
|
| 41 |
+
# Check if file exists in repo
|
| 42 |
+
files = api.list_repo_files(
|
| 43 |
+
repo_id=Settings.HF_SPACE,
|
| 44 |
+
repo_type="space"
|
| 45 |
+
)
|
| 46 |
+
if repo_path in files:
|
| 47 |
+
return True
|
| 48 |
+
|
| 49 |
+
print(f"Waiting for {file_path.name} to appear in repository... ({attempt + 1}/{max_attempts})")
|
| 50 |
+
time.sleep(10) # Wait 10 seconds between checks
|
| 51 |
+
|
| 52 |
+
except Exception as e:
|
| 53 |
+
print(f"Error checking file status: {str(e)}")
|
| 54 |
+
|
| 55 |
+
return False
|
| 56 |
+
|
| 57 |
+
|
| 58 |
def upload_large_file(api: HfApi, file_path: Path, repo_path: str):
|
| 59 |
"""Upload a single large file to the Space"""
|
| 60 |
+
file_size = file_path.stat().st_size / (1024 * 1024) # Size in MB
|
| 61 |
+
print(f"Uploading {file_path.name} ({file_size:.2f} MB)...")
|
| 62 |
|
| 63 |
# Create the commit operation
|
| 64 |
operation = CommitOperationAdd(
|
|
|
|
| 71 |
repo_id=Settings.HF_SPACE,
|
| 72 |
repo_type="space",
|
| 73 |
operations=[operation],
|
| 74 |
+
commit_message=f"Upload {file_path.name}" # Removed async flag
|
|
|
|
| 75 |
)
|
| 76 |
+
|
| 77 |
+
# Wait for file to appear in repository
|
| 78 |
+
if file_size > 100: # Only wait for files larger than 100MB
|
| 79 |
+
if wait_for_file_upload(api, file_path, repo_path):
|
| 80 |
+
print(f"Successfully uploaded and verified {file_path.name}")
|
| 81 |
+
else:
|
| 82 |
+
raise RuntimeError(f"Failed to verify upload of {file_path.name}")
|
| 83 |
+
else:
|
| 84 |
+
print(f"Successfully uploaded {file_path.name}")
|
| 85 |
|
| 86 |
|
| 87 |
def main():
|