James Edmunds commited on
Commit
fecd36c
·
1 Parent(s): 3913938

feat: add upload verification for large files

Browse files
Files changed (1) hide show
  1. scripts/upload_embeddings.py +35 -4
scripts/upload_embeddings.py CHANGED
@@ -1,6 +1,7 @@
1
  """Upload embeddings to HuggingFace Space"""
2
  import sys
3
  import os
 
4
  from pathlib import Path
5
  from huggingface_hub import HfApi, CommitOperationAdd
6
  from dotenv import load_dotenv
@@ -32,9 +33,32 @@ def verify_space_access():
32
  return False
33
 
34
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
  def upload_large_file(api: HfApi, file_path: Path, repo_path: str):
36
  """Upload a single large file to the Space"""
37
- print(f"Uploading {file_path.name}...")
 
38
 
39
  # Create the commit operation
40
  operation = CommitOperationAdd(
@@ -47,10 +71,17 @@ def upload_large_file(api: HfApi, file_path: Path, repo_path: str):
47
  repo_id=Settings.HF_SPACE,
48
  repo_type="space",
49
  operations=[operation],
50
- commit_message=f"Upload {file_path.name}",
51
- run_as_future=True # Handle large files asynchronously
52
  )
53
- print(f"Successfully uploaded {file_path.name}")
 
 
 
 
 
 
 
 
54
 
55
 
56
  def main():
 
1
  """Upload embeddings to HuggingFace Space"""
2
  import sys
3
  import os
4
+ import time
5
  from pathlib import Path
6
  from huggingface_hub import HfApi, CommitOperationAdd
7
  from dotenv import load_dotenv
 
33
  return False
34
 
35
 
36
+ def wait_for_file_upload(api: HfApi, file_path: Path, repo_path: str):
37
+ """Wait for a file to appear in the repository"""
38
+ max_attempts = 30 # 5 minutes total
39
+ for attempt in range(max_attempts):
40
+ try:
41
+ # Check if file exists in repo
42
+ files = api.list_repo_files(
43
+ repo_id=Settings.HF_SPACE,
44
+ repo_type="space"
45
+ )
46
+ if repo_path in files:
47
+ return True
48
+
49
+ print(f"Waiting for {file_path.name} to appear in repository... ({attempt + 1}/{max_attempts})")
50
+ time.sleep(10) # Wait 10 seconds between checks
51
+
52
+ except Exception as e:
53
+ print(f"Error checking file status: {str(e)}")
54
+
55
+ return False
56
+
57
+
58
  def upload_large_file(api: HfApi, file_path: Path, repo_path: str):
59
  """Upload a single large file to the Space"""
60
+ file_size = file_path.stat().st_size / (1024 * 1024) # Size in MB
61
+ print(f"Uploading {file_path.name} ({file_size:.2f} MB)...")
62
 
63
  # Create the commit operation
64
  operation = CommitOperationAdd(
 
71
  repo_id=Settings.HF_SPACE,
72
  repo_type="space",
73
  operations=[operation],
74
+ commit_message=f"Upload {file_path.name}" # Removed async flag
 
75
  )
76
+
77
+ # Wait for file to appear in repository
78
+ if file_size > 100: # Only wait for files larger than 100MB
79
+ if wait_for_file_upload(api, file_path, repo_path):
80
+ print(f"Successfully uploaded and verified {file_path.name}")
81
+ else:
82
+ raise RuntimeError(f"Failed to verify upload of {file_path.name}")
83
+ else:
84
+ print(f"Successfully uploaded {file_path.name}")
85
 
86
 
87
  def main():