Fred808 commited on
Commit
d417ed4
·
verified ·
1 Parent(s): fc8dcd0

Update main.py

Browse files
Files changed (1) hide show
  1. main.py +29 -31
main.py CHANGED
@@ -3,13 +3,12 @@ import time
3
  import json
4
  import requests
5
  from huggingface_hub import upload_file
6
- from urllib.parse import urlparse
7
 
8
  # === CONFIGURATION ===
9
  HF_TOKEN = os.environ.get("HF_TOKEN")
10
  ZEROUPLOAD_TOKEN = "vcIXZnfFfZlPg5GEw4JqeqZTroJh4nst0WqFvun37SNr1wtzwsu6wC9pZ4x8ze4z"
11
  REPO_ID = "Fred808/BG1"
12
- DATA_PATH = "AEffects"
13
  PROCESSED_FILE = "processed.json"
14
  OUTPUT_DIR = "batch_downloads"
15
 
@@ -17,21 +16,21 @@ HEADERS = {
17
  "Authorization": f"Bearer {ZEROUPLOAD_TOKEN}"
18
  }
19
 
20
- # === Only list of ZeroUpload URLs ===
21
- DOWNLOAD_URLS = [
22
- "https://zeroupload.com/e7a5f338b6db8987f1735378ad164c79"
23
  ]
24
 
25
- # === Load processed ===
26
  if os.path.exists(PROCESSED_FILE):
27
  with open(PROCESSED_FILE, "r") as f:
28
- processed_urls = set(json.load(f))
29
  else:
30
- processed_urls = set()
31
 
32
  def save_processed():
33
  with open(PROCESSED_FILE, "w") as f:
34
- json.dump(list(processed_urls), f, indent=2)
35
 
36
  def upload_to_dataset(filepath):
37
  try:
@@ -46,43 +45,42 @@ def upload_to_dataset(filepath):
46
  except Exception as e:
47
  print(f"[!] Upload failed: {filepath} — {e}")
48
 
49
- # === Prepare output directory ===
50
  os.makedirs(OUTPUT_DIR, exist_ok=True)
51
 
52
- # === Main loop ===
53
- for url in DOWNLOAD_URLS:
54
- if url in processed_urls:
55
- print(f"[✔] Already processed: {url}")
56
- continue
57
 
58
- try:
59
- print(f"[*] Requesting: {url}")
60
- with requests.get(url, headers=HEADERS, stream=True) as r:
61
- r.raise_for_status()
62
 
63
- # Try to extract filename from headers, fallback to last URL segment
64
- cd = r.headers.get("content-disposition")
65
- if cd and "filename=" in cd:
66
- filename = cd.split("filename=")[1].strip('"')
67
- else:
68
- filename = os.path.basename(urlparse(url).path) + ".mp4"
69
 
70
- local_path = os.path.join(OUTPUT_DIR, filename)
71
 
 
 
 
 
72
  with open(local_path, "wb") as f:
73
  for chunk in r.iter_content(chunk_size=8192):
74
  f.write(chunk)
75
-
76
  print(f"[✓] Downloaded: {filename}")
77
 
78
  upload_to_dataset(local_path)
79
- processed_urls.add(url)
 
80
  save_processed()
81
  os.remove(local_path)
82
 
83
  except Exception as e:
84
- print(f"[!] Error with {url}: {e}")
85
 
86
- time.sleep(2)
87
 
88
- print("\n✅ All ZeroUpload files processed.")
 
3
  import json
4
  import requests
5
  from huggingface_hub import upload_file
 
6
 
7
  # === CONFIGURATION ===
8
  HF_TOKEN = os.environ.get("HF_TOKEN")
9
  ZEROUPLOAD_TOKEN = "vcIXZnfFfZlPg5GEw4JqeqZTroJh4nst0WqFvun37SNr1wtzwsu6wC9pZ4x8ze4z"
10
  REPO_ID = "Fred808/BG1"
11
+ DATA_PATH = "Blenders"
12
  PROCESSED_FILE = "processed.json"
13
  OUTPUT_DIR = "batch_downloads"
14
 
 
16
  "Authorization": f"Bearer {ZEROUPLOAD_TOKEN}"
17
  }
18
 
19
+ # === List of files to download ===
20
+ DOWNLOAD_LIST = [
21
+ {"filename": "intro_to_blender.mp4", "url": "https://zeroupload.com/e7a5f338b6db8987f1735378ad164c79"}
22
  ]
23
 
24
+ # === Load processed files ===
25
  if os.path.exists(PROCESSED_FILE):
26
  with open(PROCESSED_FILE, "r") as f:
27
+ processed_ids = set(json.load(f))
28
  else:
29
+ processed_ids = set()
30
 
31
  def save_processed():
32
  with open(PROCESSED_FILE, "w") as f:
33
+ json.dump(list(processed_ids), f, indent=2)
34
 
35
  def upload_to_dataset(filepath):
36
  try:
 
45
  except Exception as e:
46
  print(f"[!] Upload failed: {filepath} — {e}")
47
 
48
+ # === Create output folder ===
49
  os.makedirs(OUTPUT_DIR, exist_ok=True)
50
 
51
+ # === Download and upload ===
52
+ for item in DOWNLOAD_LIST:
53
+ filename = item.get("filename")
54
+ file_url = item.get("url")
 
55
 
56
+ if not filename or not file_url:
57
+ print("[!] Skipping invalid entry.")
58
+ continue
 
59
 
60
+ if file_url in processed_ids:
61
+ print(f"[✔] Already processed: {file_url}")
62
+ continue
 
 
 
63
 
64
+ local_path = os.path.join(OUTPUT_DIR, filename)
65
 
66
+ try:
67
+ print(f"[*] Downloading {filename} from ZeroUpload...")
68
+ with requests.get(file_url, headers=HEADERS, stream=True) as r:
69
+ r.raise_for_status()
70
  with open(local_path, "wb") as f:
71
  for chunk in r.iter_content(chunk_size=8192):
72
  f.write(chunk)
 
73
  print(f"[✓] Downloaded: {filename}")
74
 
75
  upload_to_dataset(local_path)
76
+
77
+ processed_ids.add(file_url)
78
  save_processed()
79
  os.remove(local_path)
80
 
81
  except Exception as e:
82
+ print(f"[!] Error downloading/uploading {filename}: {e}")
83
 
84
+ time.sleep(2) # optional delay
85
 
86
+ print("\n✅ All downloads and uploads completed.")