sonithoiningombam commited on
Commit
4744aaa
·
verified ·
1 Parent(s): ce6f807

Pushed file: upload_workspace.py

Browse files
Files changed (1) hide show
  1. upload_workspace.py +59 -0
upload_workspace.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import time
3
+ from pathlib import Path
4
+ from huggingface_hub import upload_folder
5
+
6
+ # Setup your core configurations
7
+ FOLDER_PATH = "/teamspace/studios/this_studio"
8
+ REPO_ID = "sonithoiningombam/bart"
9
+ REPO_TYPE = "model"
10
+
11
+ # Correct patterns to completely ignore all dotfiles and hidden folders (.venv, .git, etc.)
12
+ IGNORE_PATTERNS = ["**/.*", "**/.*/**"]
13
+
14
+ print("🚀 Scanning workspace directories to chunk the upload safely...")
15
+
16
+ # Step 1: Discover top-level visible items to isolate the commits
17
+ root_path = Path(FOLDER_PATH)
18
+ items_to_upload = [
19
+ item for item in root_path.iterdir()
20
+ if not item.name.startswith('.')
21
+ ]
22
+
23
+ print(f"📦 Found {len(items_to_upload)} root items to process sequentially.")
24
+
25
+ # Step 2: Loop through items and push them to prevent commit explosion
26
+ for index, item in enumerate(items_to_upload, start=1):
27
+ print(f"\n🔄 [{index}/{len(items_to_upload)}] Uploading: {item.name}...")
28
+
29
+ try:
30
+ if item.is_file():
31
+ # If it's a standalone file in the root
32
+ upload_folder(
33
+ folder_path=str(root_path),
34
+ repo_id=REPO_ID,
35
+ repo_type=REPO_TYPE,
36
+ allow_patterns=[item.name],
37
+ commit_message=f"Pushed file: {item.name}"
38
+ )
39
+ elif item.is_dir():
40
+ # If it's a folder, upload its contents safely to its relative path
41
+ upload_folder(
42
+ folder_path=str(item),
43
+ repo_id=REPO_ID,
44
+ repo_type=REPO_TYPE,
45
+ path_in_repo=item.name,
46
+ ignore_patterns=IGNORE_PATTERNS,
47
+ commit_message=f"Pushed directory segment: {item.name}"
48
+ )
49
+ print(f"✅ Finished uploading: {item.name}")
50
+
51
+ # Step 3: Enforce a strict rest window to stay comfortably under 128 commits/hour
52
+ print("🕒 Cooling down for 30 seconds to respect Hugging Face API rate limits...")
53
+ time.sleep(30)
54
+
55
+ except Exception as e:
56
+ print(f"⚠️ Failed to upload {item.name} due to: {e}")
57
+ print("💡 The script will continue to the next item. You can re-run later to retry failed objects.")
58
+
59
+ print("\n🎉 Workspace segment loops finished successfully!")