amri12310 commited on
Commit
d3012f6
·
verified ·
1 Parent(s): 772150b

Update engine.py

Browse files
Files changed (1) hide show
  1. engine.py +240 -129
engine.py CHANGED
@@ -1,9 +1,9 @@
1
  # ⚡ CRITICAL: Force Rust Engine BEFORE importing Hugging Face
2
  import os
3
  os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
4
-
5
  import threading, shutil, json, time, glob, subprocess, urllib.parse, logging, struct, socket
6
- import warnings, requests
7
  import libtorrent as lt
8
  from flask import Flask, request, Response, stream_with_context, render_template_string, redirect, url_for, abort
9
  from huggingface_hub import HfApi
@@ -15,14 +15,17 @@ log.setLevel(logging.ERROR)
15
 
16
  app = Flask(__name__)
17
  api = HfApi()
 
18
  stream_session = requests.Session()
 
 
19
 
20
  # --- CONFIGURATION ---
21
  REPO_ID = "amri12310/hf-vault-99281-xk9-p02-mlq8-r4x-v9928-zb-v2-alpha-sync"
22
  TOKEN = os.environ.get("HF_TOKEN")
23
  GH_TOKEN = os.environ.get("GH_PAT")
24
  ADMIN_KEY = os.environ.get("ADMIN_KEY", "Amrit2026")
25
- ADDON_URL = "https://amri12310-sys-b-addon.hf.space" # ⚡ NEEDED FOR THE KILL SWITCH
26
 
27
  active_vaults = set()
28
  active_handles = {}
@@ -38,7 +41,6 @@ def check_auth():
38
  # ==========================================
39
  def fetch_cinemeta(u_id):
40
  if u_id in cinemeta_cache: return cinemeta_cache[u_id]
41
-
42
  parts = u_id.split('_')
43
  imdb_id = parts[0]
44
  is_series = len(parts) > 2 and parts[1].startswith('S')
@@ -58,7 +60,6 @@ def fetch_cinemeta(u_id):
58
  data = r.json().get("meta", {})
59
  meta["title"] = data.get("name", imdb_id)
60
  meta["poster"] = data.get("poster", meta["poster"])
61
-
62
  if is_series:
63
  s, e = int(parts[1][1:]), int(parts[2][1:])
64
  for vid in data.get("videos", []):
@@ -66,12 +67,11 @@ def fetch_cinemeta(u_id):
66
  meta["ep_name"] = vid.get("name", "")
67
  break
68
  except: pass
69
-
70
  cinemeta_cache[u_id] = meta
71
  return meta
72
 
73
  # ==========================================
74
- # FRONT-END DASHBOARD (Cinematic UI)
75
  # ==========================================
76
  DASHBOARD_HTML = """
77
  <!DOCTYPE html>
@@ -84,7 +84,7 @@ DASHBOARD_HTML = """
84
  <body class="bg-slate-900 text-gray-100 font-sans p-8">
85
  <div class="max-w-6xl mx-auto">
86
  <div class="flex justify-between items-center mb-8">
87
- <h1 class="text-3xl font-bold text-blue-400">🚀 Gold Standard Engine (Full File)</h1>
88
  <span class="text-xs font-mono text-slate-500">Authenticated Session</span>
89
  </div>
90
 
@@ -134,7 +134,6 @@ DASHBOARD_HTML = """
134
  </div>
135
  <div class="flex justify-between items-center">
136
  <span class="text-sm text-white font-bold">{{ prog }}%</span>
137
- <!-- ⚡ ROUTED TO ADDON KILL SWITCH -->
138
  <form action="{{ addon_url }}/kill/{{ id }}?key={{ key }}" method="POST">
139
  <button class="text-red-500 hover:text-red-400 text-[10px] font-bold tracking-wider hover:underline transition-colors">KILL TASK</button>
140
  </form>
@@ -194,7 +193,6 @@ def dashboard():
194
  raw_folders = sorted(list(set([f.split('/')[0] for f in files if '/' in f])))
195
  vaulted_items = [{"id": f, "meta": fetch_cinemeta(f)} for f in raw_folders]
196
  except: vaulted_items = []
197
-
198
  return render_template_string(DASHBOARD_HTML, handles=active_handles, vault_folders=vaulted_items, task_info=current_task_info, scout_status=scout_status, key=ADMIN_KEY, addon_url=ADDON_URL)
199
 
200
  # ==========================================
@@ -203,7 +201,6 @@ def dashboard():
203
  settings = {
204
  'listen_interfaces': '0.0.0.0:6881',
205
  'enable_dht': True,
206
- 'dht_bootstrap_nodes': 'router.bittorrent.com:6881,router.utorrent.com:6881,dht.transmissionbt.com:6881',
207
  'connections_limit': 350,
208
  'unchoke_slots_limit': 150,
209
  'active_downloads': 1,
@@ -213,10 +210,7 @@ settings = {
213
  'peer_connect_timeout': 8,
214
  'min_reconnect_time': 5,
215
  'max_failcount': 2,
216
- 'announce_to_all_trackers': True,
217
- 'announce_to_all_tiers': True,
218
- 'enable_outgoing_utp': False,
219
- 'enable_incoming_utp': False,
220
  }
221
  ses = lt.session(settings)
222
 
@@ -273,15 +267,36 @@ def get_status(u_id):
273
  return {"status": "completed"}, 200
274
 
275
  # ==========================================
276
- # THE GOLD STANDARD: FULL FILE ENGINE + RUST
277
  # ==========================================
278
- def do_archive(u_id, info_hash):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
279
  handle = None
280
  scout_status[u_id] = "active"
281
  try:
282
- work_dir, upload_dir = f"/app/temp/{u_id}", f"/app/upload/{u_id}"
283
- os.makedirs(work_dir, exist_ok=True); os.makedirs(upload_dir, exist_ok=True)
284
-
285
  current_task_info[u_id] = {"current_file": "Connecting...", "size_gb": "0.0", "meta": fetch_cinemeta(u_id)}
286
 
287
  print(f"[ENGINE] Fetching live HTTP trackers...", flush=True)
@@ -302,26 +317,95 @@ def do_archive(u_id, info_hash):
302
  print(f"[ENGINE] Waiting for metadata...", flush=True)
303
  while not handle.status().has_metadata:
304
  time.sleep(1)
305
- # ⚡ ABORT TRIPWIRE (Metadata Phase) ⚡
306
- if u_id not in active_vaults: raise Exception("Task Terminated by Cluster Command")
307
 
308
  info = handle.get_torrent_info()
309
- file_idx = max(range(info.num_files()), key=lambda i: info.files().file_size(i))
310
- current_task_info[u_id]["current_file"] = os.path.basename(info.files().file_path(file_idx))
311
- current_task_info[u_id]["size_gb"] = round(info.files().file_size(file_idx) / 1073741824, 2)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
312
 
313
- print(f"[ENGINE] Downloading: {handle.status().name}", flush=True)
314
  last_done = 0
315
  hard_stall_since = time.time()
316
  soft_stall_since = None
 
317
 
318
  while True:
319
- # ⚡ THE ABORT TRIPWIRE (Download Phase) ⚡
320
- if u_id not in active_vaults:
321
- print(f"[ENGINE] 🛑 Abort Signal Received! Dropping Task {u_id} immediately.", flush=True)
322
- raise Exception("Task Terminated by Cluster Command")
323
-
324
  s = handle.status()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
325
  if s.state in [lt.torrent_status.seeding, lt.torrent_status.finished]: break
326
 
327
  if s.total_done > last_done:
@@ -334,120 +418,128 @@ def do_archive(u_id, info_hash):
334
  if speed < 0.1:
335
  if soft_stall_since is None: soft_stall_since = time.time()
336
  elif time.time() - soft_stall_since > 60:
337
- print(f"[ENGINE] ⚠️ Soft-Stall detected. Re-injecting peers...", flush=True)
338
  handle.force_reannounce()
339
  threading.Thread(target=force_inject_peers, args=(info_hash, all_trackers, handle)).start()
340
  soft_stall_since = time.time()
341
  else: soft_stall_since = None
342
 
343
  done_gb, total_gb = s.total_done/1073741824, s.total_wanted/1073741824
344
- avail = getattr(s, 'distributed_copies', 0.0)
345
- disk_speed = speed
346
-
347
  if s.download_rate > 0:
348
- eta_secs = int((s.total_wanted - s.total_done) / s.download_rate)
349
- h, rem = divmod(eta_secs, 3600)
350
  m, sec = divmod(rem, 60)
351
  eta_str = f"{h:02d}h {m:02d}m {sec:02d}s"
352
  else: eta_str = "Infinite"
353
 
354
- print(f"ID: {u_id[:8]} | {s.progress*100:.1f}% ({done_gb:.2f}/{total_gb:.2f} GB) | Avail: {avail:.2f} | Seeds: {s.num_seeds} | Peers: {s.num_peers} | Speed: {speed:.2f} MB/s | Disk Speed: {disk_speed:.2f} MB/s | ETA: {eta_str}", flush=True)
355
  time.sleep(5)
356
 
357
- # ⚡ ABORT TRIPWIRE (Processing Phase) ⚡
358
- if u_id not in active_vaults: raise Exception("Task Terminated by Cluster Command")
359
 
360
- # --- LIGHTNING FAST LOCAL PROCESSING ---
361
- files = [os.path.join(r, f) for r, d, fs in os.walk(work_dir) for f in fs if f.endswith(('.mkv', '.mp4', '.avi', '.iso', '.zim', '.gguf'))]
362
- if not files: raise Exception("No media found.")
363
- video = max(files, key=os.path.getsize)
364
- v_size = os.path.getsize(video)
365
-
366
- temp_path = os.path.join(upload_dir, "proc.tmp")
367
- shutil.move(video, temp_path)
368
 
369
- print(f"[ENGINE] ⚡ Splitting {u_id} locally...", flush=True)
370
- subprocess.run(['split', '-b', '500M', '-d', '--suffix-length=2', temp_path, f"{upload_dir}/chunk_"])
371
- os.remove(temp_path)
372
-
373
- raw_chunks = sorted(glob.glob(f"{upload_dir}/chunk_*"))
374
- for i, path in enumerate(raw_chunks): os.rename(path, f"{upload_dir}/{i}.bin")
375
-
376
- with open(f"{upload_dir}/meta.json", "w") as f:
377
- json.dump({"size": v_size, "chunks": len(raw_chunks), "chunk_size_mb": 500}, f)
378
-
379
- print(f"[ENGINE] 🚀 Uploading to Vault (Powered by Rust hf_transfer)...", flush=True)
380
- # ⚡ Note: Because HF_HUB_ENABLE_HF_TRANSFER=1 is set at the top, this uses Rust under the hood!
381
- api.upload_folder(folder_path=upload_dir, path_in_repo=u_id, repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
382
- print(f"[ENGINE] ✅ Complete: {u_id}", flush=True)
383
 
384
  except Exception as e: print(f"[CRITICAL ERROR/ABORT] {e}", flush=True)
385
  finally:
386
- # ⚡ INSTANT DISK CLEANSING ⚡
387
  if handle: ses.remove_torrent(handle)
388
  active_handles.pop(u_id, None); current_task_info.pop(u_id, None); scout_status.pop(u_id, None)
389
- shutil.rmtree(work_dir, ignore_errors=True); shutil.rmtree(upload_dir, ignore_errors=True)
390
  active_vaults.discard(u_id)
391
 
392
  @app.route('/archive')
393
  def archive():
394
- u_id, info_hash = request.args.get('id'), request.args.get('hash')
 
 
 
 
 
 
395
  if u_id in active_vaults: return "Already downloading", 200
396
  active_vaults.add(u_id)
397
- threading.Thread(target=do_archive, args=(u_id, info_hash)).start()
398
  return "Started", 200
399
 
400
- # ⚡ THE ENGINE INTERNAL KILL SIGNAL ⚡
401
  @app.route('/internal_kill/<u_id>', methods=['POST'])
402
  def internal_kill(u_id):
403
  check_auth()
404
- active_vaults.discard(u_id) # Drops the flag, triggering the Tripwire in do_archive
405
  return "Node Aborted", 200
406
 
407
  # ==========================================
408
- # STREAMING FRONTEND (BACKWARDS COMPATIBLE)
409
  # ==========================================
410
  @app.route('/stream/<u_id>')
411
  def stream_vault(u_id):
412
  auth = {"Authorization": f"Bearer {TOKEN}"}
413
 
414
- # ⚡ ROUTE A: Try the NEW Gold Standard format (meta.json)
415
  try:
416
  r_meta = stream_session.get(f"https://huggingface.co/datasets/{REPO_ID}/resolve/main/{u_id}/meta.json", headers=auth, timeout=5)
417
  if r_meta.status_code == 200:
418
  meta = r_meta.json()
419
- total_size = meta.get('size', 0)
420
- num_chunks = meta.get('chunks', 0)
421
- c_size_mb = meta.get('chunk_size_mb', 500)
422
 
423
  range_header = request.headers.get('Range', None)
424
- byte_start = int(range_header.replace('bytes=', '').split('-')[0]) if range_header else 0
425
-
426
- def generate_new():
427
- chunk_size_bytes = c_size_mb * 1024 * 1024
428
- start_chunk_idx = byte_start // chunk_size_bytes
429
- for i in range(start_chunk_idx, num_chunks):
430
- chunk_url = f"https://huggingface.co/datasets/{REPO_ID}/resolve/main/{u_id}/{i}.bin"
431
- headers = auth.copy()
432
- if i == start_chunk_idx and byte_start % chunk_size_bytes > 0:
433
- headers["Range"] = f"bytes={byte_start % chunk_size_bytes}-"
434
-
435
- # ⚡ High-Performance Session & 1MB Buffer
436
- with stream_session.get(chunk_url, headers=headers, stream=True, timeout=10) as r_chunk:
437
- for data in r_chunk.iter_content(chunk_size=1024*1024):
438
- yield data
439
-
440
- headers = {
441
- "Content-Length": str(total_size - byte_start),
442
- "Content-Type": "video/x-matroska",
443
- "Accept-Ranges": "bytes",
444
- "Connection": "keep-alive" # ⚡ Keep-Alive for TV stability
445
- }
446
- if range_header: headers["Content-Range"] = f"bytes {byte_start}-{total_size-1}/{total_size}"
447
- return Response(stream_with_context(generate_new()), status=206 if range_header else 200, headers=headers)
448
- except: pass
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
449
 
450
- # ⚡ ROUTE B: Fallback to the OLD Sharded format (meta_part.json)
 
 
 
 
 
451
  all_chunks, total_size = [], 0
452
  for p in [1, 2, 3]:
453
  try:
@@ -458,42 +550,61 @@ def stream_vault(u_id):
458
  all_chunks.extend(m.get('chunks', []))
459
  except: pass
460
 
461
- if not all_chunks:
462
- return "Metadata not found (File may still be uploading)", 404
463
 
464
  all_chunks.sort(key=lambda x: x['start'])
465
  range_header = request.headers.get('Range', None)
466
  req_start = int(range_header.replace('bytes=', '').split('-')[0]) if range_header else 0
467
-
468
- def generate_old(current_start):
469
- for chunk in all_chunks:
470
- if chunk['end'] <= current_start: continue
471
- url = f"https://huggingface.co/datasets/{REPO_ID}/resolve/main/{u_id}/{chunk['name']}"
472
- headers = auth.copy()
473
- if current_start > chunk['start']:
474
- offset = current_start - chunk['start']
475
- headers["Range"] = f"bytes={offset}-"
476
-
477
- # ⚡ High-Performance Session & 1MB Buffer for Old Files Too
478
- with stream_session.get(url, headers=headers, stream=True, timeout=10) as r:
479
- if r.status_code in [200, 206]:
480
- for data in r.iter_content(chunk_size=1024*1024):
481
- yield data
482
- current_start = chunk['end']
483
- else: break
484
-
485
- headers = {
486
- "Content-Length": str(total_size - req_start),
487
- "Content-Type": "video/x-matroska",
488
- "Accept-Ranges": "bytes",
489
- "Connection": "keep-alive" # ⚡ Keep-Alive for TV stability
490
- }
491
- if range_header: headers["Content-Range"] = f"bytes {req_start}-{total_size-1}/{total_size}"
492
- return Response(stream_with_context(generate_old(req_start)), status=206 if range_header else 200, headers=headers)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
493
 
494
  @app.route('/delete_active/<u_id>', methods=['POST'])
495
  def delete_active(u_id):
496
- # Fallback local kill, but UI now routes to the Addon Kill Switch
497
  check_auth()
498
  active_vaults.discard(u_id)
499
  return redirect(url_for('dashboard', key=ADMIN_KEY))
@@ -506,4 +617,4 @@ def delete_vaulted(u_id):
506
  return redirect(url_for('dashboard', key=ADMIN_KEY))
507
 
508
  if __name__ == "__main__":
509
- app.run(host="0.0.0.0", port=7860)
 
1
  # ⚡ CRITICAL: Force Rust Engine BEFORE importing Hugging Face
2
  import os
3
  os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
4
+ import urllib3
5
  import threading, shutil, json, time, glob, subprocess, urllib.parse, logging, struct, socket
6
+ import warnings, requests, queue, re
7
  import libtorrent as lt
8
  from flask import Flask, request, Response, stream_with_context, render_template_string, redirect, url_for, abort
9
  from huggingface_hub import HfApi
 
15
 
16
  app = Flask(__name__)
17
  api = HfApi()
18
+ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
19
  stream_session = requests.Session()
20
+ adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100, max_retries=3)
21
+ stream_session.mount('https://', adapter)
22
 
23
  # --- CONFIGURATION ---
24
  REPO_ID = "amri12310/hf-vault-99281-xk9-p02-mlq8-r4x-v9928-zb-v2-alpha-sync"
25
  TOKEN = os.environ.get("HF_TOKEN")
26
  GH_TOKEN = os.environ.get("GH_PAT")
27
  ADMIN_KEY = os.environ.get("ADMIN_KEY", "Amrit2026")
28
+ ADDON_URL = "https://amri12310-sys-b-addon.hf.space"
29
 
30
  active_vaults = set()
31
  active_handles = {}
 
41
  # ==========================================
42
  def fetch_cinemeta(u_id):
43
  if u_id in cinemeta_cache: return cinemeta_cache[u_id]
 
44
  parts = u_id.split('_')
45
  imdb_id = parts[0]
46
  is_series = len(parts) > 2 and parts[1].startswith('S')
 
60
  data = r.json().get("meta", {})
61
  meta["title"] = data.get("name", imdb_id)
62
  meta["poster"] = data.get("poster", meta["poster"])
 
63
  if is_series:
64
  s, e = int(parts[1][1:]), int(parts[2][1:])
65
  for vid in data.get("videos", []):
 
67
  meta["ep_name"] = vid.get("name", "")
68
  break
69
  except: pass
 
70
  cinemeta_cache[u_id] = meta
71
  return meta
72
 
73
  # ==========================================
74
+ # FRONT-END DASHBOARD
75
  # ==========================================
76
  DASHBOARD_HTML = """
77
  <!DOCTYPE html>
 
84
  <body class="bg-slate-900 text-gray-100 font-sans p-8">
85
  <div class="max-w-6xl mx-auto">
86
  <div class="flex justify-between items-center mb-8">
87
+ <h1 class="text-3xl font-bold text-blue-400">🚀 Gold Standard Engine (Hybrid Swarm)</h1>
88
  <span class="text-xs font-mono text-slate-500">Authenticated Session</span>
89
  </div>
90
 
 
134
  </div>
135
  <div class="flex justify-between items-center">
136
  <span class="text-sm text-white font-bold">{{ prog }}%</span>
 
137
  <form action="{{ addon_url }}/kill/{{ id }}?key={{ key }}" method="POST">
138
  <button class="text-red-500 hover:text-red-400 text-[10px] font-bold tracking-wider hover:underline transition-colors">KILL TASK</button>
139
  </form>
 
193
  raw_folders = sorted(list(set([f.split('/')[0] for f in files if '/' in f])))
194
  vaulted_items = [{"id": f, "meta": fetch_cinemeta(f)} for f in raw_folders]
195
  except: vaulted_items = []
 
196
  return render_template_string(DASHBOARD_HTML, handles=active_handles, vault_folders=vaulted_items, task_info=current_task_info, scout_status=scout_status, key=ADMIN_KEY, addon_url=ADDON_URL)
197
 
198
  # ==========================================
 
201
  settings = {
202
  'listen_interfaces': '0.0.0.0:6881',
203
  'enable_dht': True,
 
204
  'connections_limit': 350,
205
  'unchoke_slots_limit': 150,
206
  'active_downloads': 1,
 
210
  'peer_connect_timeout': 8,
211
  'min_reconnect_time': 5,
212
  'max_failcount': 2,
213
+ 'announce_to_all_trackers': True
 
 
 
214
  }
215
  ses = lt.session(settings)
216
 
 
267
  return {"status": "completed"}, 200
268
 
269
  # ==========================================
270
+ # THE GOLD STANDARD: HYBRID SWARM ARCHITECTURE
271
  # ==========================================
272
+ def process_and_upload(filepath, target_u_id, start_chunk, end_chunk, chunk_size, is_movie_shard, part):
273
+ """Slices a raw file directly into memory-aligned .bin chunks and blasts to Hugging Face"""
274
+ v_size = os.path.getsize(filepath)
275
+ temp_up = f"/app/upload/{target_u_id}"
276
+ os.makedirs(temp_up, exist_ok=True)
277
+
278
+ with open(filepath, 'rb') as f:
279
+ f.seek(start_chunk * chunk_size)
280
+ for i in range(start_chunk, end_chunk + 1):
281
+ data = f.read(chunk_size)
282
+ if not data: break
283
+ with open(f"{temp_up}/{i}.bin", "wb") as out:
284
+ out.write(data)
285
+
286
+ if not is_movie_shard or part == 1:
287
+ with open(f"{temp_up}/meta.json", "w") as f:
288
+ total_chunks = (v_size // chunk_size) + (1 if v_size % chunk_size > 0 else 0)
289
+ json.dump({"size": v_size, "chunks": total_chunks, "chunk_size_mb": int(chunk_size / 1024 / 1024)}, f)
290
+
291
+ api.upload_folder(folder_path=temp_up, path_in_repo=target_u_id, repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
292
+ shutil.rmtree(temp_up, ignore_errors=True)
293
+
294
+ def do_archive(u_id, info_hash, part=1, total=1, is_series=False, priority_idx=-1):
295
  handle = None
296
  scout_status[u_id] = "active"
297
  try:
298
+ work_dir = f"/app/temp/{u_id}"
299
+ os.makedirs(work_dir, exist_ok=True)
 
300
  current_task_info[u_id] = {"current_file": "Connecting...", "size_gb": "0.0", "meta": fetch_cinemeta(u_id)}
301
 
302
  print(f"[ENGINE] Fetching live HTTP trackers...", flush=True)
 
317
  print(f"[ENGINE] Waiting for metadata...", flush=True)
318
  while not handle.status().has_metadata:
319
  time.sleep(1)
320
+ if u_id not in active_vaults: raise Exception("Task Terminated")
 
321
 
322
  info = handle.get_torrent_info()
323
+ chunk_size = 500 * 1024 * 1024
324
+ my_assigned_files = []
325
+
326
+ # ⚡==========================================⚡
327
+ # HYBRID SHARDING ENGINE: Bytes vs VIP Files
328
+ # ⚡==========================================⚡
329
+ if is_series:
330
+ # Smart VIP Filter: Enforce valid video extensions, > 10MB, no 'sample'
331
+ valid_files = []
332
+ for i in range(info.num_files()):
333
+ f_size = info.files().file_size(i)
334
+ f_name = info.files().file_path(i).lower()
335
+ if f_size > 10 * 1024 * 1024 and f_name.endswith(('.mkv', '.mp4', '.avi')) and "sample" not in f_name:
336
+ valid_files.append(i)
337
+
338
+ num_valid = len(valid_files)
339
+ files_per_node = num_valid // total if total > 0 else num_valid
340
+ start_idx = (part - 1) * files_per_node
341
+ end_idx = num_valid if part == total else part * files_per_node
342
+
343
+ my_assigned_files = valid_files[start_idx:end_idx]
344
+ file_priorities = [0] * info.num_files()
345
+
346
+ for i in my_assigned_files: file_priorities[i] = 1 # Normal priority
347
+
348
+ if priority_idx != -1 and priority_idx in my_assigned_files:
349
+ file_priorities[priority_idx] = 7 # 🔥 NITRO BOOST
350
+ print(f"🚀 [NITRO BOOST] Injecting Priority 7 for File {priority_idx}!", flush=True)
351
+
352
+ handle.prioritize_files(file_priorities)
353
+ target_file_idx = priority_idx if priority_idx != -1 else (my_assigned_files[0] if my_assigned_files else 0)
354
+ print(f"📦 [SEASON SHARD] Node {part}/{total} assigned {len(my_assigned_files)} pure video files.", flush=True)
355
+ else:
356
+ # Movie Byte Range Sharding
357
+ target_file_idx = max(range(info.num_files()), key=lambda i: info.files().file_size(i))
358
+ v_size = info.files().file_size(target_file_idx)
359
+ piece_length = info.piece_length()
360
+
361
+ total_chunks = (v_size // chunk_size) + (1 if v_size % chunk_size > 0 else 0)
362
+ chunks_per_node = total_chunks // total if total > 0 else total_chunks
363
+ start_chunk = (part - 1) * chunks_per_node
364
+ end_chunk = total_chunks if part == total else part * chunks_per_node
365
+
366
+ start_piece = (start_chunk * chunk_size) // piece_length
367
+ end_piece = ((end_chunk * chunk_size) // piece_length) + 1
368
+ if end_piece > info.num_pieces(): end_piece = info.num_pieces()
369
+
370
+ for i in range(info.num_pieces()): handle.piece_priority(i, 0)
371
+ for i in range(start_piece, end_piece): handle.piece_priority(i, 1)
372
+ print(f"🎬 [MOVIE SHARD] Node {part}/{total} assigned exact Chunks {start_chunk} to {end_chunk-1}", flush=True)
373
+
374
+ current_task_info[u_id]["current_file"] = os.path.basename(info.files().file_path(target_file_idx))
375
+ current_task_info[u_id]["size_gb"] = round(info.files().file_size(target_file_idx) / 1073741824, 2)
376
 
 
377
  last_done = 0
378
  hard_stall_since = time.time()
379
  soft_stall_since = None
380
+ processed_files = set()
381
 
382
  while True:
383
+ if u_id not in active_vaults: raise Exception("Task Terminated")
 
 
 
 
384
  s = handle.status()
385
+
386
+ # ⚡==========================================⚡
387
+ # THE EVENT-DRIVEN FLOW-THROUGH UPLOADER
388
+ # ⚡==========================================⚡
389
+ if is_series:
390
+ file_prog = handle.file_progress()
391
+ for i in my_assigned_files:
392
+ f_size = info.files().file_size(i)
393
+ if i not in processed_files and file_prog[i] == f_size:
394
+ file_path = os.path.join(work_dir, info.files().file_path(i))
395
+ if os.path.exists(file_path):
396
+ match = re.search(r'[Ss]\d+[Ee](\d+)', os.path.basename(file_path))
397
+ if not match: match = re.search(r'[^A-Za-z0-9](\d{1,2})[^A-Za-z0-9]', os.path.basename(file_path))
398
+ ep_num = int(match.group(1)) if match else i
399
+ target_u_id = re.sub(r'_E\d+_', f'_E{ep_num:02d}_', u_id) if match else f"{u_id}_F{i}"
400
+
401
+ print(f"⚡ [FLOW-THROUGH] Episode {ep_num} hit 100%. Uploading {target_u_id} instantly...", flush=True)
402
+ end_c = (f_size - 1) // chunk_size if f_size > 0 else 0
403
+ process_and_upload(file_path, target_u_id, 0, end_c, chunk_size, False, part)
404
+
405
+ try: os.remove(file_path) # Burn local file
406
+ except: pass
407
+ processed_files.add(i)
408
+
409
  if s.state in [lt.torrent_status.seeding, lt.torrent_status.finished]: break
410
 
411
  if s.total_done > last_done:
 
418
  if speed < 0.1:
419
  if soft_stall_since is None: soft_stall_since = time.time()
420
  elif time.time() - soft_stall_since > 60:
 
421
  handle.force_reannounce()
422
  threading.Thread(target=force_inject_peers, args=(info_hash, all_trackers, handle)).start()
423
  soft_stall_since = time.time()
424
  else: soft_stall_since = None
425
 
426
  done_gb, total_gb = s.total_done/1073741824, s.total_wanted/1073741824
 
 
 
427
  if s.download_rate > 0:
428
+ h, rem = divmod(int((s.total_wanted - s.total_done) / s.download_rate), 3600)
 
429
  m, sec = divmod(rem, 60)
430
  eta_str = f"{h:02d}h {m:02d}m {sec:02d}s"
431
  else: eta_str = "Infinite"
432
 
433
+ print(f"ID: {u_id[:8]} | {s.progress*100:.1f}% ({done_gb:.2f}/{total_gb:.2f} GB) | Peers: {s.num_peers} | Speed: {speed:.2f} MB/s | ETA: {eta_str}", flush=True)
434
  time.sleep(5)
435
 
436
+ if u_id not in active_vaults: raise Exception("Task Terminated")
 
437
 
438
+ # ⚡ Process Movie Shard
439
+ if not is_series:
440
+ file_path = os.path.join(work_dir, info.files().file_path(target_file_idx))
441
+ if os.path.exists(file_path):
442
+ print(f"[ENGINE] 🚀 Target Shard reached 100%. Uploading exact Byte Range to Vault ({u_id})...", flush=True)
443
+ process_and_upload(file_path, u_id, start_chunk, end_chunk - 1, chunk_size, total > 1, part)
 
 
444
 
445
+ print(f"[ENGINE] ✅ Task Complete: {u_id}", flush=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
446
 
447
  except Exception as e: print(f"[CRITICAL ERROR/ABORT] {e}", flush=True)
448
  finally:
 
449
  if handle: ses.remove_torrent(handle)
450
  active_handles.pop(u_id, None); current_task_info.pop(u_id, None); scout_status.pop(u_id, None)
451
+ shutil.rmtree(work_dir, ignore_errors=True)
452
  active_vaults.discard(u_id)
453
 
454
  @app.route('/archive')
455
  def archive():
456
+ u_id = request.args.get('id')
457
+ info_hash = request.args.get('hash')
458
+ part = int(request.args.get('part', 1))
459
+ total = int(request.args.get('total', 1))
460
+ is_series = request.args.get('is_series') == "true"
461
+ priority_idx = int(request.args.get('priority_idx', -1))
462
+
463
  if u_id in active_vaults: return "Already downloading", 200
464
  active_vaults.add(u_id)
465
+ threading.Thread(target=do_archive, args=(u_id, info_hash, part, total, is_series, priority_idx)).start()
466
  return "Started", 200
467
 
 
468
  @app.route('/internal_kill/<u_id>', methods=['POST'])
469
  def internal_kill(u_id):
470
  check_auth()
471
+ active_vaults.discard(u_id)
472
  return "Node Aborted", 200
473
 
474
  # ==========================================
475
+ # TURBO LEAD-PIPE PROXY (WITH SELF-HEALING)
476
  # ==========================================
477
  @app.route('/stream/<u_id>')
478
  def stream_vault(u_id):
479
  auth = {"Authorization": f"Bearer {TOKEN}"}
480
 
 
481
  try:
482
  r_meta = stream_session.get(f"https://huggingface.co/datasets/{REPO_ID}/resolve/main/{u_id}/meta.json", headers=auth, timeout=5)
483
  if r_meta.status_code == 200:
484
  meta = r_meta.json()
485
+ total_size, num_chunks, c_size_mb = meta.get('size', 0), meta.get('chunks', 0), meta.get('chunk_size_mb', 500)
486
+ chunk_size_bytes = c_size_mb * 1024 * 1024
 
487
 
488
  range_header = request.headers.get('Range', None)
489
+ req_start = int(range_header.replace('bytes=', '').split('-')[0]) if range_header else 0
490
+ req_end = total_size - 1
491
+ bytes_to_send = (req_end - req_start) + 1
492
+
493
+ def generate_turbo_new():
494
+ start_chunk_idx = req_start // chunk_size_bytes
495
+ first_offset = req_start % chunk_size_bytes
496
+ data_queue = queue.Queue(maxsize=400)
497
+
498
+ def turbo_worker():
499
+ try:
500
+ for i in range(start_chunk_idx, num_chunks):
501
+ current_offset = first_offset if i == start_chunk_idx else 0
502
+ while True:
503
+ chunk_url = f"https://huggingface.co/datasets/{REPO_ID}/resolve/main/{u_id}/{i}.bin"
504
+ h = auth.copy()
505
+ if current_offset > 0: h["Range"] = f"bytes={current_offset}-"
506
+ try:
507
+ with stream_session.get(chunk_url, headers=h, stream=True, timeout=(3.0, 5.0)) as r:
508
+ r.raise_for_status()
509
+ expected_len = int(r.headers.get('Content-Length', 0))
510
+ bytes_this_req = 0
511
+ for data in r.iter_content(chunk_size=1024*1024):
512
+ if data:
513
+ data_queue.put(data)
514
+ bytes_this_req += len(data)
515
+ current_offset += len(data)
516
+ if expected_len > 0 and bytes_this_req < expected_len:
517
+ print(f"⚠️ [NETWORK] HF dropped chunk {i} early. Resuming at {current_offset}...", flush=True)
518
+ continue
519
+ else: break
520
+ except Exception as e:
521
+ time.sleep(1)
522
+ except Exception as e: print(f"Turbo Worker Error: {e}", flush=True)
523
+ finally: data_queue.put(None)
524
+
525
+ threading.Thread(target=turbo_worker, daemon=True).start()
526
+
527
+ bytes_sent = 0
528
+ while bytes_sent < bytes_to_send:
529
+ chunk_data = data_queue.get()
530
+ if chunk_data is None: break
531
+ if bytes_sent + len(chunk_data) > bytes_to_send:
532
+ yield chunk_data[:bytes_to_send - bytes_sent]
533
+ break
534
+ yield chunk_data
535
+ bytes_sent += len(chunk_data)
536
 
537
+ headers = {"Content-Length": str(bytes_to_send), "Content-Type": "video/x-matroska", "Accept-Ranges": "bytes", "Connection": "keep-alive"}
538
+ if range_header: headers["Content-Range"] = f"bytes {req_start}-{req_end}/{total_size}"
539
+ return Response(stream_with_context(generate_turbo_new()), status=206 if range_header else 200, headers=headers)
540
+ except Exception as e: print(f"Meta A error: {e}", flush=True)
541
+
542
+ # ⚡ ROUTE B: OLD Sharded format
543
  all_chunks, total_size = [], 0
544
  for p in [1, 2, 3]:
545
  try:
 
550
  all_chunks.extend(m.get('chunks', []))
551
  except: pass
552
 
553
+ if not all_chunks: return "Metadata not found.", 404
 
554
 
555
  all_chunks.sort(key=lambda x: x['start'])
556
  range_header = request.headers.get('Range', None)
557
  req_start = int(range_header.replace('bytes=', '').split('-')[0]) if range_header else 0
558
+ req_end = total_size - 1
559
+ bytes_to_send = (req_end - req_start) + 1
560
+
561
+ def generate_turbo_old():
562
+ data_queue = queue.Queue(maxsize=400)
563
+ def turbo_worker_old():
564
+ current_start = req_start
565
+ try:
566
+ for chunk in all_chunks:
567
+ if chunk['end'] <= current_start: continue
568
+ while current_start < chunk['end']:
569
+ url = f"https://huggingface.co/datasets/{REPO_ID}/resolve/main/{u_id}/{chunk['name']}"
570
+ h = auth.copy()
571
+ if current_start > chunk['start']: h["Range"] = f"bytes={current_start - chunk['start']}-"
572
+ try:
573
+ with stream_session.get(url, headers=h, stream=True, timeout=(3.0, 5.0)) as r:
574
+ if r.status_code not in [200, 206]: break
575
+ expected_len = int(r.headers.get('Content-Length', 0))
576
+ bytes_this_req = 0
577
+ for data in r.iter_content(chunk_size=1024*1024):
578
+ if data:
579
+ data_queue.put(data)
580
+ current_start += len(data)
581
+ bytes_this_req += len(data)
582
+ if expected_len > 0 and bytes_this_req < expected_len:
583
+ print(f"⚠️ [NETWORK] Old chunk truncated. Resuming...", flush=True)
584
+ continue
585
+ else: break
586
+ except: time.sleep(1)
587
+ except Exception as e: print(f"Legacy Worker Error: {e}", flush=True)
588
+ finally: data_queue.put(None)
589
+
590
+ threading.Thread(target=turbo_worker_old, daemon=True).start()
591
+
592
+ bytes_sent = 0
593
+ while bytes_sent < bytes_to_send:
594
+ chunk_data = data_queue.get()
595
+ if chunk_data is None: break
596
+ if bytes_sent + len(chunk_data) > bytes_to_send:
597
+ yield chunk_data[:bytes_to_send - bytes_sent]
598
+ break
599
+ yield chunk_data
600
+ bytes_sent += len(chunk_data)
601
+
602
+ headers = {"Content-Length": str(bytes_to_send), "Content-Type": "video/x-matroska", "Accept-Ranges": "bytes", "Connection": "keep-alive"}
603
+ if range_header: headers["Content-Range"] = f"bytes {req_start}-{req_end}/{total_size}"
604
+ return Response(stream_with_context(generate_turbo_old()), status=206 if range_header else 200, headers=headers)
605
 
606
  @app.route('/delete_active/<u_id>', methods=['POST'])
607
  def delete_active(u_id):
 
608
  check_auth()
609
  active_vaults.discard(u_id)
610
  return redirect(url_for('dashboard', key=ADMIN_KEY))
 
617
  return redirect(url_for('dashboard', key=ADMIN_KEY))
618
 
619
  if __name__ == "__main__":
620
+ app.run(host="0.0.0.0", port=7860)