Spaces:
Paused
Paused
Update engine.py
Browse files
engine.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
# ⚡ CRITICAL: Force Rust Engine BEFORE importing Hugging Face
|
| 2 |
import os
|
| 3 |
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
|
| 4 |
-
|
| 5 |
import threading, shutil, json, time, glob, subprocess, urllib.parse, logging, struct, socket
|
| 6 |
-
import warnings, requests
|
| 7 |
import libtorrent as lt
|
| 8 |
from flask import Flask, request, Response, stream_with_context, render_template_string, redirect, url_for, abort
|
| 9 |
from huggingface_hub import HfApi
|
|
@@ -15,14 +15,17 @@ log.setLevel(logging.ERROR)
|
|
| 15 |
|
| 16 |
app = Flask(__name__)
|
| 17 |
api = HfApi()
|
|
|
|
| 18 |
stream_session = requests.Session()
|
|
|
|
|
|
|
| 19 |
|
| 20 |
# --- CONFIGURATION ---
|
| 21 |
REPO_ID = "amri12310/hf-vault-99281-xk9-p02-mlq8-r4x-v9928-zb-v2-alpha-sync"
|
| 22 |
TOKEN = os.environ.get("HF_TOKEN")
|
| 23 |
GH_TOKEN = os.environ.get("GH_PAT")
|
| 24 |
ADMIN_KEY = os.environ.get("ADMIN_KEY", "Amrit2026")
|
| 25 |
-
ADDON_URL = "https://amri12310-sys-b-addon.hf.space"
|
| 26 |
|
| 27 |
active_vaults = set()
|
| 28 |
active_handles = {}
|
|
@@ -38,7 +41,6 @@ def check_auth():
|
|
| 38 |
# ==========================================
|
| 39 |
def fetch_cinemeta(u_id):
|
| 40 |
if u_id in cinemeta_cache: return cinemeta_cache[u_id]
|
| 41 |
-
|
| 42 |
parts = u_id.split('_')
|
| 43 |
imdb_id = parts[0]
|
| 44 |
is_series = len(parts) > 2 and parts[1].startswith('S')
|
|
@@ -58,7 +60,6 @@ def fetch_cinemeta(u_id):
|
|
| 58 |
data = r.json().get("meta", {})
|
| 59 |
meta["title"] = data.get("name", imdb_id)
|
| 60 |
meta["poster"] = data.get("poster", meta["poster"])
|
| 61 |
-
|
| 62 |
if is_series:
|
| 63 |
s, e = int(parts[1][1:]), int(parts[2][1:])
|
| 64 |
for vid in data.get("videos", []):
|
|
@@ -66,12 +67,11 @@ def fetch_cinemeta(u_id):
|
|
| 66 |
meta["ep_name"] = vid.get("name", "")
|
| 67 |
break
|
| 68 |
except: pass
|
| 69 |
-
|
| 70 |
cinemeta_cache[u_id] = meta
|
| 71 |
return meta
|
| 72 |
|
| 73 |
# ==========================================
|
| 74 |
-
# FRONT-END DASHBOARD
|
| 75 |
# ==========================================
|
| 76 |
DASHBOARD_HTML = """
|
| 77 |
<!DOCTYPE html>
|
|
@@ -84,7 +84,7 @@ DASHBOARD_HTML = """
|
|
| 84 |
<body class="bg-slate-900 text-gray-100 font-sans p-8">
|
| 85 |
<div class="max-w-6xl mx-auto">
|
| 86 |
<div class="flex justify-between items-center mb-8">
|
| 87 |
-
<h1 class="text-3xl font-bold text-blue-400">🚀 Gold Standard Engine (
|
| 88 |
<span class="text-xs font-mono text-slate-500">Authenticated Session</span>
|
| 89 |
</div>
|
| 90 |
|
|
@@ -134,7 +134,6 @@ DASHBOARD_HTML = """
|
|
| 134 |
</div>
|
| 135 |
<div class="flex justify-between items-center">
|
| 136 |
<span class="text-sm text-white font-bold">{{ prog }}%</span>
|
| 137 |
-
<!-- ⚡ ROUTED TO ADDON KILL SWITCH -->
|
| 138 |
<form action="{{ addon_url }}/kill/{{ id }}?key={{ key }}" method="POST">
|
| 139 |
<button class="text-red-500 hover:text-red-400 text-[10px] font-bold tracking-wider hover:underline transition-colors">KILL TASK</button>
|
| 140 |
</form>
|
|
@@ -194,7 +193,6 @@ def dashboard():
|
|
| 194 |
raw_folders = sorted(list(set([f.split('/')[0] for f in files if '/' in f])))
|
| 195 |
vaulted_items = [{"id": f, "meta": fetch_cinemeta(f)} for f in raw_folders]
|
| 196 |
except: vaulted_items = []
|
| 197 |
-
|
| 198 |
return render_template_string(DASHBOARD_HTML, handles=active_handles, vault_folders=vaulted_items, task_info=current_task_info, scout_status=scout_status, key=ADMIN_KEY, addon_url=ADDON_URL)
|
| 199 |
|
| 200 |
# ==========================================
|
|
@@ -203,7 +201,6 @@ def dashboard():
|
|
| 203 |
settings = {
|
| 204 |
'listen_interfaces': '0.0.0.0:6881',
|
| 205 |
'enable_dht': True,
|
| 206 |
-
'dht_bootstrap_nodes': 'router.bittorrent.com:6881,router.utorrent.com:6881,dht.transmissionbt.com:6881',
|
| 207 |
'connections_limit': 350,
|
| 208 |
'unchoke_slots_limit': 150,
|
| 209 |
'active_downloads': 1,
|
|
@@ -213,10 +210,7 @@ settings = {
|
|
| 213 |
'peer_connect_timeout': 8,
|
| 214 |
'min_reconnect_time': 5,
|
| 215 |
'max_failcount': 2,
|
| 216 |
-
'announce_to_all_trackers': True
|
| 217 |
-
'announce_to_all_tiers': True,
|
| 218 |
-
'enable_outgoing_utp': False,
|
| 219 |
-
'enable_incoming_utp': False,
|
| 220 |
}
|
| 221 |
ses = lt.session(settings)
|
| 222 |
|
|
@@ -273,15 +267,36 @@ def get_status(u_id):
|
|
| 273 |
return {"status": "completed"}, 200
|
| 274 |
|
| 275 |
# ==========================================
|
| 276 |
-
# THE GOLD STANDARD:
|
| 277 |
# ==========================================
|
| 278 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 279 |
handle = None
|
| 280 |
scout_status[u_id] = "active"
|
| 281 |
try:
|
| 282 |
-
work_dir
|
| 283 |
-
os.makedirs(work_dir, exist_ok=True)
|
| 284 |
-
|
| 285 |
current_task_info[u_id] = {"current_file": "Connecting...", "size_gb": "0.0", "meta": fetch_cinemeta(u_id)}
|
| 286 |
|
| 287 |
print(f"[ENGINE] Fetching live HTTP trackers...", flush=True)
|
|
@@ -302,26 +317,95 @@ def do_archive(u_id, info_hash):
|
|
| 302 |
print(f"[ENGINE] Waiting for metadata...", flush=True)
|
| 303 |
while not handle.status().has_metadata:
|
| 304 |
time.sleep(1)
|
| 305 |
-
|
| 306 |
-
if u_id not in active_vaults: raise Exception("Task Terminated by Cluster Command")
|
| 307 |
|
| 308 |
info = handle.get_torrent_info()
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 312 |
|
| 313 |
-
print(f"[ENGINE] Downloading: {handle.status().name}", flush=True)
|
| 314 |
last_done = 0
|
| 315 |
hard_stall_since = time.time()
|
| 316 |
soft_stall_since = None
|
|
|
|
| 317 |
|
| 318 |
while True:
|
| 319 |
-
|
| 320 |
-
if u_id not in active_vaults:
|
| 321 |
-
print(f"[ENGINE] 🛑 Abort Signal Received! Dropping Task {u_id} immediately.", flush=True)
|
| 322 |
-
raise Exception("Task Terminated by Cluster Command")
|
| 323 |
-
|
| 324 |
s = handle.status()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 325 |
if s.state in [lt.torrent_status.seeding, lt.torrent_status.finished]: break
|
| 326 |
|
| 327 |
if s.total_done > last_done:
|
|
@@ -334,120 +418,128 @@ def do_archive(u_id, info_hash):
|
|
| 334 |
if speed < 0.1:
|
| 335 |
if soft_stall_since is None: soft_stall_since = time.time()
|
| 336 |
elif time.time() - soft_stall_since > 60:
|
| 337 |
-
print(f"[ENGINE] ⚠️ Soft-Stall detected. Re-injecting peers...", flush=True)
|
| 338 |
handle.force_reannounce()
|
| 339 |
threading.Thread(target=force_inject_peers, args=(info_hash, all_trackers, handle)).start()
|
| 340 |
soft_stall_since = time.time()
|
| 341 |
else: soft_stall_since = None
|
| 342 |
|
| 343 |
done_gb, total_gb = s.total_done/1073741824, s.total_wanted/1073741824
|
| 344 |
-
avail = getattr(s, 'distributed_copies', 0.0)
|
| 345 |
-
disk_speed = speed
|
| 346 |
-
|
| 347 |
if s.download_rate > 0:
|
| 348 |
-
|
| 349 |
-
h, rem = divmod(eta_secs, 3600)
|
| 350 |
m, sec = divmod(rem, 60)
|
| 351 |
eta_str = f"{h:02d}h {m:02d}m {sec:02d}s"
|
| 352 |
else: eta_str = "Infinite"
|
| 353 |
|
| 354 |
-
print(f"ID: {u_id[:8]} | {s.progress*100:.1f}% ({done_gb:.2f}/{total_gb:.2f} GB) |
|
| 355 |
time.sleep(5)
|
| 356 |
|
| 357 |
-
|
| 358 |
-
if u_id not in active_vaults: raise Exception("Task Terminated by Cluster Command")
|
| 359 |
|
| 360 |
-
#
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
temp_path = os.path.join(upload_dir, "proc.tmp")
|
| 367 |
-
shutil.move(video, temp_path)
|
| 368 |
|
| 369 |
-
print(f"[ENGINE]
|
| 370 |
-
subprocess.run(['split', '-b', '500M', '-d', '--suffix-length=2', temp_path, f"{upload_dir}/chunk_"])
|
| 371 |
-
os.remove(temp_path)
|
| 372 |
-
|
| 373 |
-
raw_chunks = sorted(glob.glob(f"{upload_dir}/chunk_*"))
|
| 374 |
-
for i, path in enumerate(raw_chunks): os.rename(path, f"{upload_dir}/{i}.bin")
|
| 375 |
-
|
| 376 |
-
with open(f"{upload_dir}/meta.json", "w") as f:
|
| 377 |
-
json.dump({"size": v_size, "chunks": len(raw_chunks), "chunk_size_mb": 500}, f)
|
| 378 |
-
|
| 379 |
-
print(f"[ENGINE] 🚀 Uploading to Vault (Powered by Rust hf_transfer)...", flush=True)
|
| 380 |
-
# ⚡ Note: Because HF_HUB_ENABLE_HF_TRANSFER=1 is set at the top, this uses Rust under the hood!
|
| 381 |
-
api.upload_folder(folder_path=upload_dir, path_in_repo=u_id, repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
|
| 382 |
-
print(f"[ENGINE] ✅ Complete: {u_id}", flush=True)
|
| 383 |
|
| 384 |
except Exception as e: print(f"[CRITICAL ERROR/ABORT] {e}", flush=True)
|
| 385 |
finally:
|
| 386 |
-
# ⚡ INSTANT DISK CLEANSING ⚡
|
| 387 |
if handle: ses.remove_torrent(handle)
|
| 388 |
active_handles.pop(u_id, None); current_task_info.pop(u_id, None); scout_status.pop(u_id, None)
|
| 389 |
-
shutil.rmtree(work_dir, ignore_errors=True)
|
| 390 |
active_vaults.discard(u_id)
|
| 391 |
|
| 392 |
@app.route('/archive')
|
| 393 |
def archive():
|
| 394 |
-
u_id
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 395 |
if u_id in active_vaults: return "Already downloading", 200
|
| 396 |
active_vaults.add(u_id)
|
| 397 |
-
threading.Thread(target=do_archive, args=(u_id, info_hash)).start()
|
| 398 |
return "Started", 200
|
| 399 |
|
| 400 |
-
# ⚡ THE ENGINE INTERNAL KILL SIGNAL ⚡
|
| 401 |
@app.route('/internal_kill/<u_id>', methods=['POST'])
|
| 402 |
def internal_kill(u_id):
|
| 403 |
check_auth()
|
| 404 |
-
active_vaults.discard(u_id)
|
| 405 |
return "Node Aborted", 200
|
| 406 |
|
| 407 |
# ==========================================
|
| 408 |
-
#
|
| 409 |
# ==========================================
|
| 410 |
@app.route('/stream/<u_id>')
|
| 411 |
def stream_vault(u_id):
|
| 412 |
auth = {"Authorization": f"Bearer {TOKEN}"}
|
| 413 |
|
| 414 |
-
# ⚡ ROUTE A: Try the NEW Gold Standard format (meta.json)
|
| 415 |
try:
|
| 416 |
r_meta = stream_session.get(f"https://huggingface.co/datasets/{REPO_ID}/resolve/main/{u_id}/meta.json", headers=auth, timeout=5)
|
| 417 |
if r_meta.status_code == 200:
|
| 418 |
meta = r_meta.json()
|
| 419 |
-
total_size = meta.get('size', 0)
|
| 420 |
-
|
| 421 |
-
c_size_mb = meta.get('chunk_size_mb', 500)
|
| 422 |
|
| 423 |
range_header = request.headers.get('Range', None)
|
| 424 |
-
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
|
| 444 |
-
|
| 445 |
-
|
| 446 |
-
|
| 447 |
-
|
| 448 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 449 |
|
| 450 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 451 |
all_chunks, total_size = [], 0
|
| 452 |
for p in [1, 2, 3]:
|
| 453 |
try:
|
|
@@ -458,42 +550,61 @@ def stream_vault(u_id):
|
|
| 458 |
all_chunks.extend(m.get('chunks', []))
|
| 459 |
except: pass
|
| 460 |
|
| 461 |
-
if not all_chunks:
|
| 462 |
-
return "Metadata not found (File may still be uploading)", 404
|
| 463 |
|
| 464 |
all_chunks.sort(key=lambda x: x['start'])
|
| 465 |
range_header = request.headers.get('Range', None)
|
| 466 |
req_start = int(range_header.replace('bytes=', '').split('-')[0]) if range_header else 0
|
| 467 |
-
|
| 468 |
-
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
|
| 473 |
-
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
|
| 477 |
-
|
| 478 |
-
|
| 479 |
-
|
| 480 |
-
|
| 481 |
-
|
| 482 |
-
|
| 483 |
-
|
| 484 |
-
|
| 485 |
-
|
| 486 |
-
|
| 487 |
-
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
|
| 491 |
-
|
| 492 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 493 |
|
| 494 |
@app.route('/delete_active/<u_id>', methods=['POST'])
|
| 495 |
def delete_active(u_id):
|
| 496 |
-
# Fallback local kill, but UI now routes to the Addon Kill Switch
|
| 497 |
check_auth()
|
| 498 |
active_vaults.discard(u_id)
|
| 499 |
return redirect(url_for('dashboard', key=ADMIN_KEY))
|
|
@@ -506,4 +617,4 @@ def delete_vaulted(u_id):
|
|
| 506 |
return redirect(url_for('dashboard', key=ADMIN_KEY))
|
| 507 |
|
| 508 |
if __name__ == "__main__":
|
| 509 |
-
app.run(host="0.0.0.0", port=7860)
|
|
|
|
| 1 |
# ⚡ CRITICAL: Force Rust Engine BEFORE importing Hugging Face
|
| 2 |
import os
|
| 3 |
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
|
| 4 |
+
import urllib3
|
| 5 |
import threading, shutil, json, time, glob, subprocess, urllib.parse, logging, struct, socket
|
| 6 |
+
import warnings, requests, queue, re
|
| 7 |
import libtorrent as lt
|
| 8 |
from flask import Flask, request, Response, stream_with_context, render_template_string, redirect, url_for, abort
|
| 9 |
from huggingface_hub import HfApi
|
|
|
|
| 15 |
|
| 16 |
app = Flask(__name__)
|
| 17 |
api = HfApi()
|
| 18 |
+
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
| 19 |
stream_session = requests.Session()
|
| 20 |
+
adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100, max_retries=3)
|
| 21 |
+
stream_session.mount('https://', adapter)
|
| 22 |
|
| 23 |
# --- CONFIGURATION ---
|
| 24 |
REPO_ID = "amri12310/hf-vault-99281-xk9-p02-mlq8-r4x-v9928-zb-v2-alpha-sync"
|
| 25 |
TOKEN = os.environ.get("HF_TOKEN")
|
| 26 |
GH_TOKEN = os.environ.get("GH_PAT")
|
| 27 |
ADMIN_KEY = os.environ.get("ADMIN_KEY", "Amrit2026")
|
| 28 |
+
ADDON_URL = "https://amri12310-sys-b-addon.hf.space"
|
| 29 |
|
| 30 |
active_vaults = set()
|
| 31 |
active_handles = {}
|
|
|
|
| 41 |
# ==========================================
|
| 42 |
def fetch_cinemeta(u_id):
|
| 43 |
if u_id in cinemeta_cache: return cinemeta_cache[u_id]
|
|
|
|
| 44 |
parts = u_id.split('_')
|
| 45 |
imdb_id = parts[0]
|
| 46 |
is_series = len(parts) > 2 and parts[1].startswith('S')
|
|
|
|
| 60 |
data = r.json().get("meta", {})
|
| 61 |
meta["title"] = data.get("name", imdb_id)
|
| 62 |
meta["poster"] = data.get("poster", meta["poster"])
|
|
|
|
| 63 |
if is_series:
|
| 64 |
s, e = int(parts[1][1:]), int(parts[2][1:])
|
| 65 |
for vid in data.get("videos", []):
|
|
|
|
| 67 |
meta["ep_name"] = vid.get("name", "")
|
| 68 |
break
|
| 69 |
except: pass
|
|
|
|
| 70 |
cinemeta_cache[u_id] = meta
|
| 71 |
return meta
|
| 72 |
|
| 73 |
# ==========================================
|
| 74 |
+
# FRONT-END DASHBOARD
|
| 75 |
# ==========================================
|
| 76 |
DASHBOARD_HTML = """
|
| 77 |
<!DOCTYPE html>
|
|
|
|
| 84 |
<body class="bg-slate-900 text-gray-100 font-sans p-8">
|
| 85 |
<div class="max-w-6xl mx-auto">
|
| 86 |
<div class="flex justify-between items-center mb-8">
|
| 87 |
+
<h1 class="text-3xl font-bold text-blue-400">🚀 Gold Standard Engine (Hybrid Swarm)</h1>
|
| 88 |
<span class="text-xs font-mono text-slate-500">Authenticated Session</span>
|
| 89 |
</div>
|
| 90 |
|
|
|
|
| 134 |
</div>
|
| 135 |
<div class="flex justify-between items-center">
|
| 136 |
<span class="text-sm text-white font-bold">{{ prog }}%</span>
|
|
|
|
| 137 |
<form action="{{ addon_url }}/kill/{{ id }}?key={{ key }}" method="POST">
|
| 138 |
<button class="text-red-500 hover:text-red-400 text-[10px] font-bold tracking-wider hover:underline transition-colors">KILL TASK</button>
|
| 139 |
</form>
|
|
|
|
| 193 |
raw_folders = sorted(list(set([f.split('/')[0] for f in files if '/' in f])))
|
| 194 |
vaulted_items = [{"id": f, "meta": fetch_cinemeta(f)} for f in raw_folders]
|
| 195 |
except: vaulted_items = []
|
|
|
|
| 196 |
return render_template_string(DASHBOARD_HTML, handles=active_handles, vault_folders=vaulted_items, task_info=current_task_info, scout_status=scout_status, key=ADMIN_KEY, addon_url=ADDON_URL)
|
| 197 |
|
| 198 |
# ==========================================
|
|
|
|
| 201 |
settings = {
|
| 202 |
'listen_interfaces': '0.0.0.0:6881',
|
| 203 |
'enable_dht': True,
|
|
|
|
| 204 |
'connections_limit': 350,
|
| 205 |
'unchoke_slots_limit': 150,
|
| 206 |
'active_downloads': 1,
|
|
|
|
| 210 |
'peer_connect_timeout': 8,
|
| 211 |
'min_reconnect_time': 5,
|
| 212 |
'max_failcount': 2,
|
| 213 |
+
'announce_to_all_trackers': True
|
|
|
|
|
|
|
|
|
|
| 214 |
}
|
| 215 |
ses = lt.session(settings)
|
| 216 |
|
|
|
|
| 267 |
return {"status": "completed"}, 200
|
| 268 |
|
| 269 |
# ==========================================
|
| 270 |
+
# THE GOLD STANDARD: HYBRID SWARM ARCHITECTURE
|
| 271 |
# ==========================================
|
| 272 |
+
def process_and_upload(filepath, target_u_id, start_chunk, end_chunk, chunk_size, is_movie_shard, part):
|
| 273 |
+
"""Slices a raw file directly into memory-aligned .bin chunks and blasts to Hugging Face"""
|
| 274 |
+
v_size = os.path.getsize(filepath)
|
| 275 |
+
temp_up = f"/app/upload/{target_u_id}"
|
| 276 |
+
os.makedirs(temp_up, exist_ok=True)
|
| 277 |
+
|
| 278 |
+
with open(filepath, 'rb') as f:
|
| 279 |
+
f.seek(start_chunk * chunk_size)
|
| 280 |
+
for i in range(start_chunk, end_chunk + 1):
|
| 281 |
+
data = f.read(chunk_size)
|
| 282 |
+
if not data: break
|
| 283 |
+
with open(f"{temp_up}/{i}.bin", "wb") as out:
|
| 284 |
+
out.write(data)
|
| 285 |
+
|
| 286 |
+
if not is_movie_shard or part == 1:
|
| 287 |
+
with open(f"{temp_up}/meta.json", "w") as f:
|
| 288 |
+
total_chunks = (v_size // chunk_size) + (1 if v_size % chunk_size > 0 else 0)
|
| 289 |
+
json.dump({"size": v_size, "chunks": total_chunks, "chunk_size_mb": int(chunk_size / 1024 / 1024)}, f)
|
| 290 |
+
|
| 291 |
+
api.upload_folder(folder_path=temp_up, path_in_repo=target_u_id, repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
|
| 292 |
+
shutil.rmtree(temp_up, ignore_errors=True)
|
| 293 |
+
|
| 294 |
+
def do_archive(u_id, info_hash, part=1, total=1, is_series=False, priority_idx=-1):
|
| 295 |
handle = None
|
| 296 |
scout_status[u_id] = "active"
|
| 297 |
try:
|
| 298 |
+
work_dir = f"/app/temp/{u_id}"
|
| 299 |
+
os.makedirs(work_dir, exist_ok=True)
|
|
|
|
| 300 |
current_task_info[u_id] = {"current_file": "Connecting...", "size_gb": "0.0", "meta": fetch_cinemeta(u_id)}
|
| 301 |
|
| 302 |
print(f"[ENGINE] Fetching live HTTP trackers...", flush=True)
|
|
|
|
| 317 |
print(f"[ENGINE] Waiting for metadata...", flush=True)
|
| 318 |
while not handle.status().has_metadata:
|
| 319 |
time.sleep(1)
|
| 320 |
+
if u_id not in active_vaults: raise Exception("Task Terminated")
|
|
|
|
| 321 |
|
| 322 |
info = handle.get_torrent_info()
|
| 323 |
+
chunk_size = 500 * 1024 * 1024
|
| 324 |
+
my_assigned_files = []
|
| 325 |
+
|
| 326 |
+
# ⚡==========================================⚡
|
| 327 |
+
# HYBRID SHARDING ENGINE: Bytes vs VIP Files
|
| 328 |
+
# ⚡==========================================⚡
|
| 329 |
+
if is_series:
|
| 330 |
+
# Smart VIP Filter: Enforce valid video extensions, > 10MB, no 'sample'
|
| 331 |
+
valid_files = []
|
| 332 |
+
for i in range(info.num_files()):
|
| 333 |
+
f_size = info.files().file_size(i)
|
| 334 |
+
f_name = info.files().file_path(i).lower()
|
| 335 |
+
if f_size > 10 * 1024 * 1024 and f_name.endswith(('.mkv', '.mp4', '.avi')) and "sample" not in f_name:
|
| 336 |
+
valid_files.append(i)
|
| 337 |
+
|
| 338 |
+
num_valid = len(valid_files)
|
| 339 |
+
files_per_node = num_valid // total if total > 0 else num_valid
|
| 340 |
+
start_idx = (part - 1) * files_per_node
|
| 341 |
+
end_idx = num_valid if part == total else part * files_per_node
|
| 342 |
+
|
| 343 |
+
my_assigned_files = valid_files[start_idx:end_idx]
|
| 344 |
+
file_priorities = [0] * info.num_files()
|
| 345 |
+
|
| 346 |
+
for i in my_assigned_files: file_priorities[i] = 1 # Normal priority
|
| 347 |
+
|
| 348 |
+
if priority_idx != -1 and priority_idx in my_assigned_files:
|
| 349 |
+
file_priorities[priority_idx] = 7 # 🔥 NITRO BOOST
|
| 350 |
+
print(f"🚀 [NITRO BOOST] Injecting Priority 7 for File {priority_idx}!", flush=True)
|
| 351 |
+
|
| 352 |
+
handle.prioritize_files(file_priorities)
|
| 353 |
+
target_file_idx = priority_idx if priority_idx != -1 else (my_assigned_files[0] if my_assigned_files else 0)
|
| 354 |
+
print(f"📦 [SEASON SHARD] Node {part}/{total} assigned {len(my_assigned_files)} pure video files.", flush=True)
|
| 355 |
+
else:
|
| 356 |
+
# Movie Byte Range Sharding
|
| 357 |
+
target_file_idx = max(range(info.num_files()), key=lambda i: info.files().file_size(i))
|
| 358 |
+
v_size = info.files().file_size(target_file_idx)
|
| 359 |
+
piece_length = info.piece_length()
|
| 360 |
+
|
| 361 |
+
total_chunks = (v_size // chunk_size) + (1 if v_size % chunk_size > 0 else 0)
|
| 362 |
+
chunks_per_node = total_chunks // total if total > 0 else total_chunks
|
| 363 |
+
start_chunk = (part - 1) * chunks_per_node
|
| 364 |
+
end_chunk = total_chunks if part == total else part * chunks_per_node
|
| 365 |
+
|
| 366 |
+
start_piece = (start_chunk * chunk_size) // piece_length
|
| 367 |
+
end_piece = ((end_chunk * chunk_size) // piece_length) + 1
|
| 368 |
+
if end_piece > info.num_pieces(): end_piece = info.num_pieces()
|
| 369 |
+
|
| 370 |
+
for i in range(info.num_pieces()): handle.piece_priority(i, 0)
|
| 371 |
+
for i in range(start_piece, end_piece): handle.piece_priority(i, 1)
|
| 372 |
+
print(f"🎬 [MOVIE SHARD] Node {part}/{total} assigned exact Chunks {start_chunk} to {end_chunk-1}", flush=True)
|
| 373 |
+
|
| 374 |
+
current_task_info[u_id]["current_file"] = os.path.basename(info.files().file_path(target_file_idx))
|
| 375 |
+
current_task_info[u_id]["size_gb"] = round(info.files().file_size(target_file_idx) / 1073741824, 2)
|
| 376 |
|
|
|
|
| 377 |
last_done = 0
|
| 378 |
hard_stall_since = time.time()
|
| 379 |
soft_stall_since = None
|
| 380 |
+
processed_files = set()
|
| 381 |
|
| 382 |
while True:
|
| 383 |
+
if u_id not in active_vaults: raise Exception("Task Terminated")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 384 |
s = handle.status()
|
| 385 |
+
|
| 386 |
+
# ⚡==========================================⚡
|
| 387 |
+
# THE EVENT-DRIVEN FLOW-THROUGH UPLOADER
|
| 388 |
+
# ⚡==========================================⚡
|
| 389 |
+
if is_series:
|
| 390 |
+
file_prog = handle.file_progress()
|
| 391 |
+
for i in my_assigned_files:
|
| 392 |
+
f_size = info.files().file_size(i)
|
| 393 |
+
if i not in processed_files and file_prog[i] == f_size:
|
| 394 |
+
file_path = os.path.join(work_dir, info.files().file_path(i))
|
| 395 |
+
if os.path.exists(file_path):
|
| 396 |
+
match = re.search(r'[Ss]\d+[Ee](\d+)', os.path.basename(file_path))
|
| 397 |
+
if not match: match = re.search(r'[^A-Za-z0-9](\d{1,2})[^A-Za-z0-9]', os.path.basename(file_path))
|
| 398 |
+
ep_num = int(match.group(1)) if match else i
|
| 399 |
+
target_u_id = re.sub(r'_E\d+_', f'_E{ep_num:02d}_', u_id) if match else f"{u_id}_F{i}"
|
| 400 |
+
|
| 401 |
+
print(f"⚡ [FLOW-THROUGH] Episode {ep_num} hit 100%. Uploading {target_u_id} instantly...", flush=True)
|
| 402 |
+
end_c = (f_size - 1) // chunk_size if f_size > 0 else 0
|
| 403 |
+
process_and_upload(file_path, target_u_id, 0, end_c, chunk_size, False, part)
|
| 404 |
+
|
| 405 |
+
try: os.remove(file_path) # Burn local file
|
| 406 |
+
except: pass
|
| 407 |
+
processed_files.add(i)
|
| 408 |
+
|
| 409 |
if s.state in [lt.torrent_status.seeding, lt.torrent_status.finished]: break
|
| 410 |
|
| 411 |
if s.total_done > last_done:
|
|
|
|
| 418 |
if speed < 0.1:
|
| 419 |
if soft_stall_since is None: soft_stall_since = time.time()
|
| 420 |
elif time.time() - soft_stall_since > 60:
|
|
|
|
| 421 |
handle.force_reannounce()
|
| 422 |
threading.Thread(target=force_inject_peers, args=(info_hash, all_trackers, handle)).start()
|
| 423 |
soft_stall_since = time.time()
|
| 424 |
else: soft_stall_since = None
|
| 425 |
|
| 426 |
done_gb, total_gb = s.total_done/1073741824, s.total_wanted/1073741824
|
|
|
|
|
|
|
|
|
|
| 427 |
if s.download_rate > 0:
|
| 428 |
+
h, rem = divmod(int((s.total_wanted - s.total_done) / s.download_rate), 3600)
|
|
|
|
| 429 |
m, sec = divmod(rem, 60)
|
| 430 |
eta_str = f"{h:02d}h {m:02d}m {sec:02d}s"
|
| 431 |
else: eta_str = "Infinite"
|
| 432 |
|
| 433 |
+
print(f"ID: {u_id[:8]} | {s.progress*100:.1f}% ({done_gb:.2f}/{total_gb:.2f} GB) | Peers: {s.num_peers} | Speed: {speed:.2f} MB/s | ETA: {eta_str}", flush=True)
|
| 434 |
time.sleep(5)
|
| 435 |
|
| 436 |
+
if u_id not in active_vaults: raise Exception("Task Terminated")
|
|
|
|
| 437 |
|
| 438 |
+
# ⚡ Process Movie Shard
|
| 439 |
+
if not is_series:
|
| 440 |
+
file_path = os.path.join(work_dir, info.files().file_path(target_file_idx))
|
| 441 |
+
if os.path.exists(file_path):
|
| 442 |
+
print(f"[ENGINE] 🚀 Target Shard reached 100%. Uploading exact Byte Range to Vault ({u_id})...", flush=True)
|
| 443 |
+
process_and_upload(file_path, u_id, start_chunk, end_chunk - 1, chunk_size, total > 1, part)
|
|
|
|
|
|
|
| 444 |
|
| 445 |
+
print(f"[ENGINE] ✅ Task Complete: {u_id}", flush=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 446 |
|
| 447 |
except Exception as e: print(f"[CRITICAL ERROR/ABORT] {e}", flush=True)
|
| 448 |
finally:
|
|
|
|
| 449 |
if handle: ses.remove_torrent(handle)
|
| 450 |
active_handles.pop(u_id, None); current_task_info.pop(u_id, None); scout_status.pop(u_id, None)
|
| 451 |
+
shutil.rmtree(work_dir, ignore_errors=True)
|
| 452 |
active_vaults.discard(u_id)
|
| 453 |
|
| 454 |
@app.route('/archive')
|
| 455 |
def archive():
|
| 456 |
+
u_id = request.args.get('id')
|
| 457 |
+
info_hash = request.args.get('hash')
|
| 458 |
+
part = int(request.args.get('part', 1))
|
| 459 |
+
total = int(request.args.get('total', 1))
|
| 460 |
+
is_series = request.args.get('is_series') == "true"
|
| 461 |
+
priority_idx = int(request.args.get('priority_idx', -1))
|
| 462 |
+
|
| 463 |
if u_id in active_vaults: return "Already downloading", 200
|
| 464 |
active_vaults.add(u_id)
|
| 465 |
+
threading.Thread(target=do_archive, args=(u_id, info_hash, part, total, is_series, priority_idx)).start()
|
| 466 |
return "Started", 200
|
| 467 |
|
|
|
|
| 468 |
@app.route('/internal_kill/<u_id>', methods=['POST'])
|
| 469 |
def internal_kill(u_id):
|
| 470 |
check_auth()
|
| 471 |
+
active_vaults.discard(u_id)
|
| 472 |
return "Node Aborted", 200
|
| 473 |
|
| 474 |
# ==========================================
|
| 475 |
+
# TURBO LEAD-PIPE PROXY (WITH SELF-HEALING)
|
| 476 |
# ==========================================
|
| 477 |
@app.route('/stream/<u_id>')
|
| 478 |
def stream_vault(u_id):
|
| 479 |
auth = {"Authorization": f"Bearer {TOKEN}"}
|
| 480 |
|
|
|
|
| 481 |
try:
|
| 482 |
r_meta = stream_session.get(f"https://huggingface.co/datasets/{REPO_ID}/resolve/main/{u_id}/meta.json", headers=auth, timeout=5)
|
| 483 |
if r_meta.status_code == 200:
|
| 484 |
meta = r_meta.json()
|
| 485 |
+
total_size, num_chunks, c_size_mb = meta.get('size', 0), meta.get('chunks', 0), meta.get('chunk_size_mb', 500)
|
| 486 |
+
chunk_size_bytes = c_size_mb * 1024 * 1024
|
|
|
|
| 487 |
|
| 488 |
range_header = request.headers.get('Range', None)
|
| 489 |
+
req_start = int(range_header.replace('bytes=', '').split('-')[0]) if range_header else 0
|
| 490 |
+
req_end = total_size - 1
|
| 491 |
+
bytes_to_send = (req_end - req_start) + 1
|
| 492 |
+
|
| 493 |
+
def generate_turbo_new():
|
| 494 |
+
start_chunk_idx = req_start // chunk_size_bytes
|
| 495 |
+
first_offset = req_start % chunk_size_bytes
|
| 496 |
+
data_queue = queue.Queue(maxsize=400)
|
| 497 |
+
|
| 498 |
+
def turbo_worker():
|
| 499 |
+
try:
|
| 500 |
+
for i in range(start_chunk_idx, num_chunks):
|
| 501 |
+
current_offset = first_offset if i == start_chunk_idx else 0
|
| 502 |
+
while True:
|
| 503 |
+
chunk_url = f"https://huggingface.co/datasets/{REPO_ID}/resolve/main/{u_id}/{i}.bin"
|
| 504 |
+
h = auth.copy()
|
| 505 |
+
if current_offset > 0: h["Range"] = f"bytes={current_offset}-"
|
| 506 |
+
try:
|
| 507 |
+
with stream_session.get(chunk_url, headers=h, stream=True, timeout=(3.0, 5.0)) as r:
|
| 508 |
+
r.raise_for_status()
|
| 509 |
+
expected_len = int(r.headers.get('Content-Length', 0))
|
| 510 |
+
bytes_this_req = 0
|
| 511 |
+
for data in r.iter_content(chunk_size=1024*1024):
|
| 512 |
+
if data:
|
| 513 |
+
data_queue.put(data)
|
| 514 |
+
bytes_this_req += len(data)
|
| 515 |
+
current_offset += len(data)
|
| 516 |
+
if expected_len > 0 and bytes_this_req < expected_len:
|
| 517 |
+
print(f"⚠️ [NETWORK] HF dropped chunk {i} early. Resuming at {current_offset}...", flush=True)
|
| 518 |
+
continue
|
| 519 |
+
else: break
|
| 520 |
+
except Exception as e:
|
| 521 |
+
time.sleep(1)
|
| 522 |
+
except Exception as e: print(f"Turbo Worker Error: {e}", flush=True)
|
| 523 |
+
finally: data_queue.put(None)
|
| 524 |
+
|
| 525 |
+
threading.Thread(target=turbo_worker, daemon=True).start()
|
| 526 |
+
|
| 527 |
+
bytes_sent = 0
|
| 528 |
+
while bytes_sent < bytes_to_send:
|
| 529 |
+
chunk_data = data_queue.get()
|
| 530 |
+
if chunk_data is None: break
|
| 531 |
+
if bytes_sent + len(chunk_data) > bytes_to_send:
|
| 532 |
+
yield chunk_data[:bytes_to_send - bytes_sent]
|
| 533 |
+
break
|
| 534 |
+
yield chunk_data
|
| 535 |
+
bytes_sent += len(chunk_data)
|
| 536 |
|
| 537 |
+
headers = {"Content-Length": str(bytes_to_send), "Content-Type": "video/x-matroska", "Accept-Ranges": "bytes", "Connection": "keep-alive"}
|
| 538 |
+
if range_header: headers["Content-Range"] = f"bytes {req_start}-{req_end}/{total_size}"
|
| 539 |
+
return Response(stream_with_context(generate_turbo_new()), status=206 if range_header else 200, headers=headers)
|
| 540 |
+
except Exception as e: print(f"Meta A error: {e}", flush=True)
|
| 541 |
+
|
| 542 |
+
# ⚡ ROUTE B: OLD Sharded format
|
| 543 |
all_chunks, total_size = [], 0
|
| 544 |
for p in [1, 2, 3]:
|
| 545 |
try:
|
|
|
|
| 550 |
all_chunks.extend(m.get('chunks', []))
|
| 551 |
except: pass
|
| 552 |
|
| 553 |
+
if not all_chunks: return "Metadata not found.", 404
|
|
|
|
| 554 |
|
| 555 |
all_chunks.sort(key=lambda x: x['start'])
|
| 556 |
range_header = request.headers.get('Range', None)
|
| 557 |
req_start = int(range_header.replace('bytes=', '').split('-')[0]) if range_header else 0
|
| 558 |
+
req_end = total_size - 1
|
| 559 |
+
bytes_to_send = (req_end - req_start) + 1
|
| 560 |
+
|
| 561 |
+
def generate_turbo_old():
|
| 562 |
+
data_queue = queue.Queue(maxsize=400)
|
| 563 |
+
def turbo_worker_old():
|
| 564 |
+
current_start = req_start
|
| 565 |
+
try:
|
| 566 |
+
for chunk in all_chunks:
|
| 567 |
+
if chunk['end'] <= current_start: continue
|
| 568 |
+
while current_start < chunk['end']:
|
| 569 |
+
url = f"https://huggingface.co/datasets/{REPO_ID}/resolve/main/{u_id}/{chunk['name']}"
|
| 570 |
+
h = auth.copy()
|
| 571 |
+
if current_start > chunk['start']: h["Range"] = f"bytes={current_start - chunk['start']}-"
|
| 572 |
+
try:
|
| 573 |
+
with stream_session.get(url, headers=h, stream=True, timeout=(3.0, 5.0)) as r:
|
| 574 |
+
if r.status_code not in [200, 206]: break
|
| 575 |
+
expected_len = int(r.headers.get('Content-Length', 0))
|
| 576 |
+
bytes_this_req = 0
|
| 577 |
+
for data in r.iter_content(chunk_size=1024*1024):
|
| 578 |
+
if data:
|
| 579 |
+
data_queue.put(data)
|
| 580 |
+
current_start += len(data)
|
| 581 |
+
bytes_this_req += len(data)
|
| 582 |
+
if expected_len > 0 and bytes_this_req < expected_len:
|
| 583 |
+
print(f"⚠️ [NETWORK] Old chunk truncated. Resuming...", flush=True)
|
| 584 |
+
continue
|
| 585 |
+
else: break
|
| 586 |
+
except: time.sleep(1)
|
| 587 |
+
except Exception as e: print(f"Legacy Worker Error: {e}", flush=True)
|
| 588 |
+
finally: data_queue.put(None)
|
| 589 |
+
|
| 590 |
+
threading.Thread(target=turbo_worker_old, daemon=True).start()
|
| 591 |
+
|
| 592 |
+
bytes_sent = 0
|
| 593 |
+
while bytes_sent < bytes_to_send:
|
| 594 |
+
chunk_data = data_queue.get()
|
| 595 |
+
if chunk_data is None: break
|
| 596 |
+
if bytes_sent + len(chunk_data) > bytes_to_send:
|
| 597 |
+
yield chunk_data[:bytes_to_send - bytes_sent]
|
| 598 |
+
break
|
| 599 |
+
yield chunk_data
|
| 600 |
+
bytes_sent += len(chunk_data)
|
| 601 |
+
|
| 602 |
+
headers = {"Content-Length": str(bytes_to_send), "Content-Type": "video/x-matroska", "Accept-Ranges": "bytes", "Connection": "keep-alive"}
|
| 603 |
+
if range_header: headers["Content-Range"] = f"bytes {req_start}-{req_end}/{total_size}"
|
| 604 |
+
return Response(stream_with_context(generate_turbo_old()), status=206 if range_header else 200, headers=headers)
|
| 605 |
|
| 606 |
@app.route('/delete_active/<u_id>', methods=['POST'])
|
| 607 |
def delete_active(u_id):
|
|
|
|
| 608 |
check_auth()
|
| 609 |
active_vaults.discard(u_id)
|
| 610 |
return redirect(url_for('dashboard', key=ADMIN_KEY))
|
|
|
|
| 617 |
return redirect(url_for('dashboard', key=ADMIN_KEY))
|
| 618 |
|
| 619 |
if __name__ == "__main__":
|
| 620 |
+
app.run(host="0.0.0.0", port=7860)
|