amri12310 commited on
Commit
8e4f0d1
·
verified ·
1 Parent(s): 99fc34b

Update engine.py

Browse files
Files changed (1) hide show
  1. engine.py +119 -61
engine.py CHANGED
@@ -1,10 +1,9 @@
1
- import os, threading, shutil, json, time, glob, subprocess, urllib.parse, logging
2
  import warnings, requests, struct, socket
3
  import libtorrent as lt
4
  from flask import Flask, request, Response, stream_with_context, render_template_string, redirect, url_for, abort
5
  from huggingface_hub import HfApi
6
 
7
- # --- PRE-FLIGHT ---
8
  warnings.filterwarnings("ignore", category=DeprecationWarning)
9
  log = logging.getLogger('werkzeug')
10
  log.setLevel(logging.ERROR)
@@ -12,31 +11,27 @@ log.setLevel(logging.ERROR)
12
  app = Flask(__name__)
13
  api = HfApi()
14
 
15
- # --- CONFIG ---
16
  REPO_ID = "amri12310/hf-vault-99281-xk9-p02-mlq8-r4x-v9928-zb-v2-alpha-sync"
17
  TOKEN = os.environ.get("HF_TOKEN")
18
- GH_TOKEN = os.environ.get("GH_PAT")
19
- ADMIN_KEY = os.environ.get("ADMIN_KEY", "default_key")
20
 
21
  active_vaults = set()
22
  active_handles = {}
23
- scout_status = {}
24
  current_task_info = {}
25
 
26
- # ==========================================
27
- # SECURITY & DASHBOARD
28
- # ==========================================
29
  def check_auth():
30
- if request.args.get('key') != ADMIN_KEY:
31
- abort(403)
32
 
 
 
 
33
  DASHBOARD_HTML = """
34
  <!DOCTYPE html>
35
  <html>
36
  <head>
37
- <title>Muscle Engine | Command Center</title>
38
  <script src="https://cdn.tailwindcss.com"></script>
39
- <meta http-equiv="refresh" content="15">
40
  </head>
41
  <body class="bg-slate-900 text-gray-100 font-sans p-8">
42
  <div class="max-w-6xl mx-auto">
@@ -56,12 +51,12 @@ DASHBOARD_HTML = """
56
  <span class="text-blue-300">{{ id[:12] }}...</span>
57
  <span class="text-slate-400">{{ (s.download_rate / 1048576)|round(2) }} MB/s</span>
58
  </div>
59
- <div class="text-xs mb-2 text-slate-300 italic truncate">Target: {{ task.get('current_file') }}</div>
60
  <div class="w-full bg-slate-700 rounded-full h-4 overflow-hidden">
61
  <div class="bg-blue-500 h-full transition-all" style="width: {{ task.get('progress', 0) }}%"></div>
62
  </div>
63
  <div class="flex justify-between mt-2">
64
- <span class="text-xs text-slate-500">Shard Part {{ task.get('part') }} of {{ task.get('total') }}</span>
65
  <form action="/delete_active/{{ id }}?key={{ key }}" method="POST">
66
  <button class="text-red-500 text-xs font-bold hover:underline">KILL TASK</button>
67
  </form>
@@ -84,66 +79,98 @@ def dashboard():
84
  return render_template_string(DASHBOARD_HTML, handles=active_handles, task_info=current_task_info, key=ADMIN_KEY)
85
 
86
  # ==========================================
87
- # CORE ENGINE LOGIC
88
  # ==========================================
89
  settings = {'listen_interfaces': '0.0.0.0:6881', 'enable_dht': True, 'aio_threads': 16, 'cache_size': 65536}
90
  ses = lt.session(settings)
91
 
92
- def do_archive(u_id, info_hash, part, total):
93
  handle = None
94
  work_dir = f"/app/temp/{u_id}"
 
 
 
95
  os.makedirs(work_dir, exist_ok=True)
 
96
  try:
 
97
  atp = lt.parse_magnet_uri(f"magnet:?xt=urn:btih:{info_hash}")
98
  atp.save_path, atp.storage_mode = work_dir, lt.storage_mode_t.storage_mode_sparse
99
  handle = ses.add_torrent(atp)
100
  active_handles[u_id] = handle
101
 
 
 
 
 
102
  while not handle.status().has_metadata: time.sleep(1)
103
 
104
  info = handle.get_torrent_info()
105
- all_files = [i for i in range(info.num_files()) if info.files().file_path(i).lower().endswith(('.mkv', '.mp4', '.avi')) and 'sample' not in info.files().file_path(i).lower()]
106
 
107
- # Sharding
108
- chunk_size = len(all_files) // total
109
- my_shard_files = all_files[(part-1)*chunk_size : (part*chunk_size if part < total else len(all_files))]
110
 
111
- master_meta = {}
112
- for loop_idx, file_idx in enumerate(my_shard_files):
113
- f_path = info.files().file_path(file_idx)
114
- f_name, f_size = os.path.basename(f_path), info.files().file_size(file_idx)
115
- current_task_info[u_id] = {"current_file": f_name, "part": part, "total": total, "progress": 0}
116
-
117
- # Prioritize
118
- for i in range(info.num_files()):
119
- handle.file_priority(i, 7 if i == file_idx else (1 if loop_idx+1 < len(my_shard_files) and i == my_shard_files[loop_idx+1] else 0))
120
-
121
- # Download Loop
122
- last_done, stall_time = 0, time.time()
123
- while True:
124
- done = handle.file_progress()[file_idx]
125
- current_task_info[u_id]['progress'] = round((done/f_size)*100, 2)
126
- if done == f_size: break
127
- if done > last_done: last_done, stall_time = done, time.time()
128
- elif time.time() - stall_time > 600: scout_status[u_id] = "stalled"
129
- time.sleep(10)
 
 
 
130
 
131
- # Chunk-and-Destroy Upload
132
- local_full = os.path.join(work_dir, f_path)
133
- c_idx = 0
134
- with open(local_full, "rb") as f_in:
135
- while True:
136
- chunk = f_in.read(500 * 1024 * 1024)
137
- if not chunk: break
138
- api.upload_file(path_or_fileobj=chunk, path_in_repo=f"{u_id}/{file_idx}_{c_idx}.bin", repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
139
- c_idx += 1
140
- master_meta[f_name] = {"size": f_size, "chunks": c_idx, "id": file_idx}
141
- os.remove(local_full)
142
-
143
- with open(f"{work_dir}/meta.json", "w") as m: json.dump(master_meta, m)
144
- api.upload_file(path_or_fileobj=f"{work_dir}/meta.json", path_in_repo=f"{u_id}/meta_{part}.json", repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
145
-
146
- except Exception as e: print(f"ERROR: {e}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
147
  finally:
148
  if handle: ses.remove_torrent(handle)
149
  active_handles.pop(u_id, None); current_task_info.pop(u_id, None); active_vaults.discard(u_id)
@@ -153,18 +180,49 @@ def do_archive(u_id, info_hash, part, total):
153
  def archive():
154
  u_id, info_hash = request.args.get('id'), request.args.get('hash')
155
  part, total = int(request.args.get('part', 1)), int(request.args.get('total', 1))
156
-
157
  print(f"[API ROUTE] Received command: Shard {part} of {total} for {u_id}", flush=True)
158
 
159
  if u_id not in active_vaults:
160
  active_vaults.add(u_id)
161
- threading.Thread(target=do_archive, args=(u_id, info_hash, part, total)).start()
162
  return "Sniper Dispatched", 200
 
 
163
  return "Busy", 200
164
 
165
- @app.route('/status/<u_id>')
166
- def get_status(u_id):
167
- return {"status": "active" if u_id in active_vaults else "completed"}, 200
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
168
 
169
  @app.route('/delete_active/<u_id>', methods=['POST'])
170
  def delete_active(u_id):
 
1
+ import os, threading, shutil, json, time, glob, urllib.parse, logging, traceback
2
  import warnings, requests, struct, socket
3
  import libtorrent as lt
4
  from flask import Flask, request, Response, stream_with_context, render_template_string, redirect, url_for, abort
5
  from huggingface_hub import HfApi
6
 
 
7
  warnings.filterwarnings("ignore", category=DeprecationWarning)
8
  log = logging.getLogger('werkzeug')
9
  log.setLevel(logging.ERROR)
 
11
  app = Flask(__name__)
12
  api = HfApi()
13
 
 
14
  REPO_ID = "amri12310/hf-vault-99281-xk9-p02-mlq8-r4x-v9928-zb-v2-alpha-sync"
15
  TOKEN = os.environ.get("HF_TOKEN")
16
+ ADMIN_KEY = os.environ.get("ADMIN_KEY", "Amrit2026")
 
17
 
18
  active_vaults = set()
19
  active_handles = {}
 
20
  current_task_info = {}
21
 
 
 
 
22
  def check_auth():
23
+ if request.args.get('key') != ADMIN_KEY: abort(403)
 
24
 
25
+ # ==========================================
26
+ # DASHBOARD HTML
27
+ # ==========================================
28
  DASHBOARD_HTML = """
29
  <!DOCTYPE html>
30
  <html>
31
  <head>
32
+ <title>System B | Command Center</title>
33
  <script src="https://cdn.tailwindcss.com"></script>
34
+ <meta http-equiv="refresh" content="10">
35
  </head>
36
  <body class="bg-slate-900 text-gray-100 font-sans p-8">
37
  <div class="max-w-6xl mx-auto">
 
51
  <span class="text-blue-300">{{ id[:12] }}...</span>
52
  <span class="text-slate-400">{{ (s.download_rate / 1048576)|round(2) }} MB/s</span>
53
  </div>
54
+ <div class="text-xs mb-2 text-slate-300 italic truncate">Target: {{ task.get('current_file', 'Awaiting Metadata...') }}</div>
55
  <div class="w-full bg-slate-700 rounded-full h-4 overflow-hidden">
56
  <div class="bg-blue-500 h-full transition-all" style="width: {{ task.get('progress', 0) }}%"></div>
57
  </div>
58
  <div class="flex justify-between mt-2">
59
+ <span class="text-xs text-slate-500">Shard Part {{ task.get('part', '?') }} of {{ task.get('total', '?') }}</span>
60
  <form action="/delete_active/{{ id }}?key={{ key }}" method="POST">
61
  <button class="text-red-500 text-xs font-bold hover:underline">KILL TASK</button>
62
  </form>
 
79
  return render_template_string(DASHBOARD_HTML, handles=active_handles, task_info=current_task_info, key=ADMIN_KEY)
80
 
81
  # ==========================================
82
+ # PIECE-LEVEL SHARDING ENGINE
83
  # ==========================================
84
  settings = {'listen_interfaces': '0.0.0.0:6881', 'enable_dht': True, 'aio_threads': 16, 'cache_size': 65536}
85
  ses = lt.session(settings)
86
 
87
+ def do_piece_shard(u_id, info_hash, part, total):
88
  handle = None
89
  work_dir = f"/app/temp/{u_id}"
90
+
91
+ # ⚡ FIX: AGGRESSIVE JUNK CLEANUP BEFORE STARTING ⚡
92
+ shutil.rmtree(work_dir, ignore_errors=True)
93
  os.makedirs(work_dir, exist_ok=True)
94
+
95
  try:
96
+ print(f"[ENGINE] Shard {part}: Starting thread for {u_id}", flush=True)
97
  atp = lt.parse_magnet_uri(f"magnet:?xt=urn:btih:{info_hash}")
98
  atp.save_path, atp.storage_mode = work_dir, lt.storage_mode_t.storage_mode_sparse
99
  handle = ses.add_torrent(atp)
100
  active_handles[u_id] = handle
101
 
102
+ # UI Placeholder
103
+ current_task_info[u_id] = {"current_file": "Awaiting Metadata...", "part": part, "total": total, "progress": 0}
104
+
105
+ print(f"[ENGINE] Shard {part}: Awaiting Metadata...", flush=True)
106
  while not handle.status().has_metadata: time.sleep(1)
107
 
108
  info = handle.get_torrent_info()
 
109
 
110
+ file_idx = max(range(info.num_files()), key=lambda i: info.files().file_size(i))
111
+ f_size = info.files().file_size(file_idx)
112
+ f_name = os.path.basename(info.files().file_path(file_idx))
113
 
114
+ pr_start = info.map_file(file_idx, 0, 1).piece
115
+ pr_end = info.map_file(file_idx, f_size - 1, 1).piece
116
+ total_pieces = pr_end - pr_start + 1
117
+ pieces_per_shard = total_pieces // total
118
+
119
+ my_start_piece = pr_start + (part - 1) * pieces_per_shard
120
+ my_end_piece = my_start_piece + pieces_per_shard if part < total else pr_end + 1
121
+
122
+ print(f"[ENGINE] Shard {part}: Sniping Pieces {my_start_piece} to {my_end_piece-1}", flush=True)
123
+
124
+ for i in range(info.num_pieces()):
125
+ if my_start_piece <= i < my_end_piece: handle.piece_priority(i, 7)
126
+ else: handle.piece_priority(i, 0)
127
+
128
+ current_task_info[u_id] = {"current_file": f_name, "part": part, "total": total, "progress": 0}
129
+
130
+ pieces_wanted = my_end_piece - my_start_piece
131
+ while True:
132
+ s = handle.status()
133
+ pieces_done = sum(1 for i in range(my_start_piece, my_end_piece) if handle.have_piece(i))
134
+ prog = (pieces_done / pieces_wanted) * 100 if pieces_wanted > 0 else 100
135
+ current_task_info[u_id]['progress'] = round(prog, 2)
136
 
137
+ if pieces_done >= pieces_wanted: break
138
+ time.sleep(5)
139
+
140
+ print(f"[ENGINE] 🚀 Uploading Shard {part}...", flush=True)
141
+ chunk_size_bytes = 500 * 1024 * 1024
142
+
143
+ byte_start = (my_start_piece - pr_start) * info.piece_length()
144
+ byte_end = f_size if part == total else (my_end_piece - pr_start) * info.piece_length()
145
+ starting_chunk_index = byte_start // chunk_size_bytes
146
+
147
+ local_full = os.path.join(work_dir, info.files().file_path(file_idx))
148
+
149
+ current_chunk_idx = starting_chunk_index
150
+ bytes_read = 0
151
+ total_bytes_to_read = byte_end - byte_start
152
+
153
+ with open(local_full, "rb") as f_in:
154
+ f_in.seek(byte_start)
155
+ while bytes_read < total_bytes_to_read:
156
+ to_read = min(chunk_size_bytes, total_bytes_to_read - bytes_read)
157
+ chunk = f_in.read(to_read)
158
+ if not chunk: break
159
+
160
+ api.upload_file(path_or_fileobj=chunk, path_in_repo=f"{u_id}/{current_chunk_idx}.bin", repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
161
+ bytes_read += len(chunk)
162
+ current_chunk_idx += 1
163
+
164
+ meta_data = {"size": f_size, "chunk_size_mb": 500, "total_chunks": (f_size // chunk_size_bytes) + 1}
165
+ with open(f"{work_dir}/meta_part{part}.json", "w") as m: json.dump(meta_data, m)
166
+ api.upload_file(path_or_fileobj=f"{work_dir}/meta_part{part}.json", path_in_repo=f"{u_id}/meta_part{part}.json", repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
167
+
168
+ os.remove(local_full)
169
+ print(f"[ENGINE] ✅ Shard {part} Complete.", flush=True)
170
+
171
+ # ⚡ FIX: BULLETPROOF ERROR LOGGING ⚡
172
+ except Exception as e:
173
+ print(f"\n[CRITICAL ERROR in Shard {part}] {e}\n{traceback.format_exc()}\n", flush=True)
174
  finally:
175
  if handle: ses.remove_torrent(handle)
176
  active_handles.pop(u_id, None); current_task_info.pop(u_id, None); active_vaults.discard(u_id)
 
180
  def archive():
181
  u_id, info_hash = request.args.get('id'), request.args.get('hash')
182
  part, total = int(request.args.get('part', 1)), int(request.args.get('total', 1))
183
+
184
  print(f"[API ROUTE] Received command: Shard {part} of {total} for {u_id}", flush=True)
185
 
186
  if u_id not in active_vaults:
187
  active_vaults.add(u_id)
188
+ threading.Thread(target=do_piece_shard, args=(u_id, info_hash, part, total)).start()
189
  return "Sniper Dispatched", 200
190
+
191
+ print(f"[API ROUTE] Busy. Ignored.", flush=True)
192
  return "Busy", 200
193
 
194
+ # ==========================================
195
+ # COMBINED STREAM PROXY
196
+ # ==========================================
197
+ @app.route('/stream/<u_id>')
198
+ def stream_vault(u_id):
199
+ auth = {"Authorization": f"Bearer {TOKEN}"}
200
+ meta = None
201
+ for p in [1, 2]:
202
+ try:
203
+ r = requests.get(f"https://huggingface.co/datasets/{REPO_ID}/resolve/main/{u_id}/meta_part{p}.json", headers=auth, timeout=5)
204
+ if r.status_code == 200: meta = r.json(); break
205
+ except: pass
206
+ if not meta: return "File not vaulted yet.", 404
207
+
208
+ total_size, c_size_mb, num_chunks = meta.get('size', 0), meta.get('chunk_size_mb', 500), meta.get('total_chunks', 0)
209
+ range_header = request.headers.get('Range', None)
210
+ byte_start = int(range_header.replace('bytes=', '').split('-')[0]) if range_header else 0
211
+
212
+ def generate():
213
+ c_bytes = c_size_mb * 1024 * 1024
214
+ for i in range(byte_start // c_bytes, num_chunks):
215
+ url = f"https://huggingface.co/datasets/{REPO_ID}/resolve/main/{u_id}/{i}.bin"
216
+ headers = auth.copy()
217
+ if i == (byte_start // c_bytes) and byte_start % c_bytes > 0: headers["Range"] = f"bytes={byte_start % c_bytes}-"
218
+ with requests.get(url, headers=headers, stream=True) as r:
219
+ if r.status_code in [200, 206]:
220
+ for data in r.iter_content(chunk_size=1024*512): yield data
221
+ else: break
222
+
223
+ headers = {"Content-Length": str(total_size - byte_start), "Content-Type": "video/x-matroska", "Accept-Ranges": "bytes"}
224
+ if range_header: headers["Content-Range"] = f"bytes {byte_start}-{total_size-1}/{total_size}"
225
+ return Response(stream_with_context(generate()), status=206 if range_header else 200, headers=headers)
226
 
227
  @app.route('/delete_active/<u_id>', methods=['POST'])
228
  def delete_active(u_id):