Spaces:
Paused
Paused
Update engine.py
Browse files
engine.py
CHANGED
|
@@ -1,10 +1,9 @@
|
|
| 1 |
-
import os, threading, shutil, json, time, glob,
|
| 2 |
import warnings, requests, struct, socket
|
| 3 |
import libtorrent as lt
|
| 4 |
from flask import Flask, request, Response, stream_with_context, render_template_string, redirect, url_for, abort
|
| 5 |
from huggingface_hub import HfApi
|
| 6 |
|
| 7 |
-
# --- PRE-FLIGHT ---
|
| 8 |
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
| 9 |
log = logging.getLogger('werkzeug')
|
| 10 |
log.setLevel(logging.ERROR)
|
|
@@ -12,31 +11,27 @@ log.setLevel(logging.ERROR)
|
|
| 12 |
app = Flask(__name__)
|
| 13 |
api = HfApi()
|
| 14 |
|
| 15 |
-
# --- CONFIG ---
|
| 16 |
REPO_ID = "amri12310/hf-vault-99281-xk9-p02-mlq8-r4x-v9928-zb-v2-alpha-sync"
|
| 17 |
TOKEN = os.environ.get("HF_TOKEN")
|
| 18 |
-
|
| 19 |
-
ADMIN_KEY = os.environ.get("ADMIN_KEY", "default_key")
|
| 20 |
|
| 21 |
active_vaults = set()
|
| 22 |
active_handles = {}
|
| 23 |
-
scout_status = {}
|
| 24 |
current_task_info = {}
|
| 25 |
|
| 26 |
-
# ==========================================
|
| 27 |
-
# SECURITY & DASHBOARD
|
| 28 |
-
# ==========================================
|
| 29 |
def check_auth():
|
| 30 |
-
if request.args.get('key') != ADMIN_KEY:
|
| 31 |
-
abort(403)
|
| 32 |
|
|
|
|
|
|
|
|
|
|
| 33 |
DASHBOARD_HTML = """
|
| 34 |
<!DOCTYPE html>
|
| 35 |
<html>
|
| 36 |
<head>
|
| 37 |
-
<title>
|
| 38 |
<script src="https://cdn.tailwindcss.com"></script>
|
| 39 |
-
<meta http-equiv="refresh" content="
|
| 40 |
</head>
|
| 41 |
<body class="bg-slate-900 text-gray-100 font-sans p-8">
|
| 42 |
<div class="max-w-6xl mx-auto">
|
|
@@ -56,12 +51,12 @@ DASHBOARD_HTML = """
|
|
| 56 |
<span class="text-blue-300">{{ id[:12] }}...</span>
|
| 57 |
<span class="text-slate-400">{{ (s.download_rate / 1048576)|round(2) }} MB/s</span>
|
| 58 |
</div>
|
| 59 |
-
<div class="text-xs mb-2 text-slate-300 italic truncate">Target: {{ task.get('current_file') }}</div>
|
| 60 |
<div class="w-full bg-slate-700 rounded-full h-4 overflow-hidden">
|
| 61 |
<div class="bg-blue-500 h-full transition-all" style="width: {{ task.get('progress', 0) }}%"></div>
|
| 62 |
</div>
|
| 63 |
<div class="flex justify-between mt-2">
|
| 64 |
-
<span class="text-xs text-slate-500">Shard Part {{ task.get('part') }} of {{ task.get('total') }}</span>
|
| 65 |
<form action="/delete_active/{{ id }}?key={{ key }}" method="POST">
|
| 66 |
<button class="text-red-500 text-xs font-bold hover:underline">KILL TASK</button>
|
| 67 |
</form>
|
|
@@ -84,66 +79,98 @@ def dashboard():
|
|
| 84 |
return render_template_string(DASHBOARD_HTML, handles=active_handles, task_info=current_task_info, key=ADMIN_KEY)
|
| 85 |
|
| 86 |
# ==========================================
|
| 87 |
-
#
|
| 88 |
# ==========================================
|
| 89 |
settings = {'listen_interfaces': '0.0.0.0:6881', 'enable_dht': True, 'aio_threads': 16, 'cache_size': 65536}
|
| 90 |
ses = lt.session(settings)
|
| 91 |
|
| 92 |
-
def
|
| 93 |
handle = None
|
| 94 |
work_dir = f"/app/temp/{u_id}"
|
|
|
|
|
|
|
|
|
|
| 95 |
os.makedirs(work_dir, exist_ok=True)
|
|
|
|
| 96 |
try:
|
|
|
|
| 97 |
atp = lt.parse_magnet_uri(f"magnet:?xt=urn:btih:{info_hash}")
|
| 98 |
atp.save_path, atp.storage_mode = work_dir, lt.storage_mode_t.storage_mode_sparse
|
| 99 |
handle = ses.add_torrent(atp)
|
| 100 |
active_handles[u_id] = handle
|
| 101 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
while not handle.status().has_metadata: time.sleep(1)
|
| 103 |
|
| 104 |
info = handle.get_torrent_info()
|
| 105 |
-
all_files = [i for i in range(info.num_files()) if info.files().file_path(i).lower().endswith(('.mkv', '.mp4', '.avi')) and 'sample' not in info.files().file_path(i).lower()]
|
| 106 |
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
|
|
|
|
|
|
|
|
|
| 130 |
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 147 |
finally:
|
| 148 |
if handle: ses.remove_torrent(handle)
|
| 149 |
active_handles.pop(u_id, None); current_task_info.pop(u_id, None); active_vaults.discard(u_id)
|
|
@@ -153,18 +180,49 @@ def do_archive(u_id, info_hash, part, total):
|
|
| 153 |
def archive():
|
| 154 |
u_id, info_hash = request.args.get('id'), request.args.get('hash')
|
| 155 |
part, total = int(request.args.get('part', 1)), int(request.args.get('total', 1))
|
| 156 |
-
|
| 157 |
print(f"[API ROUTE] Received command: Shard {part} of {total} for {u_id}", flush=True)
|
| 158 |
|
| 159 |
if u_id not in active_vaults:
|
| 160 |
active_vaults.add(u_id)
|
| 161 |
-
threading.Thread(target=
|
| 162 |
return "Sniper Dispatched", 200
|
|
|
|
|
|
|
| 163 |
return "Busy", 200
|
| 164 |
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
|
| 169 |
@app.route('/delete_active/<u_id>', methods=['POST'])
|
| 170 |
def delete_active(u_id):
|
|
|
|
| 1 |
+
import os, threading, shutil, json, time, glob, urllib.parse, logging, traceback
|
| 2 |
import warnings, requests, struct, socket
|
| 3 |
import libtorrent as lt
|
| 4 |
from flask import Flask, request, Response, stream_with_context, render_template_string, redirect, url_for, abort
|
| 5 |
from huggingface_hub import HfApi
|
| 6 |
|
|
|
|
| 7 |
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
| 8 |
log = logging.getLogger('werkzeug')
|
| 9 |
log.setLevel(logging.ERROR)
|
|
|
|
| 11 |
app = Flask(__name__)
|
| 12 |
api = HfApi()
|
| 13 |
|
|
|
|
| 14 |
REPO_ID = "amri12310/hf-vault-99281-xk9-p02-mlq8-r4x-v9928-zb-v2-alpha-sync"
|
| 15 |
TOKEN = os.environ.get("HF_TOKEN")
|
| 16 |
+
ADMIN_KEY = os.environ.get("ADMIN_KEY", "Amrit2026")
|
|
|
|
| 17 |
|
| 18 |
active_vaults = set()
|
| 19 |
active_handles = {}
|
|
|
|
| 20 |
current_task_info = {}
|
| 21 |
|
|
|
|
|
|
|
|
|
|
| 22 |
def check_auth():
|
| 23 |
+
if request.args.get('key') != ADMIN_KEY: abort(403)
|
|
|
|
| 24 |
|
| 25 |
+
# ==========================================
|
| 26 |
+
# DASHBOARD HTML
|
| 27 |
+
# ==========================================
|
| 28 |
DASHBOARD_HTML = """
|
| 29 |
<!DOCTYPE html>
|
| 30 |
<html>
|
| 31 |
<head>
|
| 32 |
+
<title>System B | Command Center</title>
|
| 33 |
<script src="https://cdn.tailwindcss.com"></script>
|
| 34 |
+
<meta http-equiv="refresh" content="10">
|
| 35 |
</head>
|
| 36 |
<body class="bg-slate-900 text-gray-100 font-sans p-8">
|
| 37 |
<div class="max-w-6xl mx-auto">
|
|
|
|
| 51 |
<span class="text-blue-300">{{ id[:12] }}...</span>
|
| 52 |
<span class="text-slate-400">{{ (s.download_rate / 1048576)|round(2) }} MB/s</span>
|
| 53 |
</div>
|
| 54 |
+
<div class="text-xs mb-2 text-slate-300 italic truncate">Target: {{ task.get('current_file', 'Awaiting Metadata...') }}</div>
|
| 55 |
<div class="w-full bg-slate-700 rounded-full h-4 overflow-hidden">
|
| 56 |
<div class="bg-blue-500 h-full transition-all" style="width: {{ task.get('progress', 0) }}%"></div>
|
| 57 |
</div>
|
| 58 |
<div class="flex justify-between mt-2">
|
| 59 |
+
<span class="text-xs text-slate-500">Shard Part {{ task.get('part', '?') }} of {{ task.get('total', '?') }}</span>
|
| 60 |
<form action="/delete_active/{{ id }}?key={{ key }}" method="POST">
|
| 61 |
<button class="text-red-500 text-xs font-bold hover:underline">KILL TASK</button>
|
| 62 |
</form>
|
|
|
|
| 79 |
return render_template_string(DASHBOARD_HTML, handles=active_handles, task_info=current_task_info, key=ADMIN_KEY)
|
| 80 |
|
| 81 |
# ==========================================
|
| 82 |
+
# PIECE-LEVEL SHARDING ENGINE
|
| 83 |
# ==========================================
|
| 84 |
settings = {'listen_interfaces': '0.0.0.0:6881', 'enable_dht': True, 'aio_threads': 16, 'cache_size': 65536}
|
| 85 |
ses = lt.session(settings)
|
| 86 |
|
| 87 |
+
def do_piece_shard(u_id, info_hash, part, total):
|
| 88 |
handle = None
|
| 89 |
work_dir = f"/app/temp/{u_id}"
|
| 90 |
+
|
| 91 |
+
# ⚡ FIX: AGGRESSIVE JUNK CLEANUP BEFORE STARTING ⚡
|
| 92 |
+
shutil.rmtree(work_dir, ignore_errors=True)
|
| 93 |
os.makedirs(work_dir, exist_ok=True)
|
| 94 |
+
|
| 95 |
try:
|
| 96 |
+
print(f"[ENGINE] Shard {part}: Starting thread for {u_id}", flush=True)
|
| 97 |
atp = lt.parse_magnet_uri(f"magnet:?xt=urn:btih:{info_hash}")
|
| 98 |
atp.save_path, atp.storage_mode = work_dir, lt.storage_mode_t.storage_mode_sparse
|
| 99 |
handle = ses.add_torrent(atp)
|
| 100 |
active_handles[u_id] = handle
|
| 101 |
|
| 102 |
+
# UI Placeholder
|
| 103 |
+
current_task_info[u_id] = {"current_file": "Awaiting Metadata...", "part": part, "total": total, "progress": 0}
|
| 104 |
+
|
| 105 |
+
print(f"[ENGINE] Shard {part}: Awaiting Metadata...", flush=True)
|
| 106 |
while not handle.status().has_metadata: time.sleep(1)
|
| 107 |
|
| 108 |
info = handle.get_torrent_info()
|
|
|
|
| 109 |
|
| 110 |
+
file_idx = max(range(info.num_files()), key=lambda i: info.files().file_size(i))
|
| 111 |
+
f_size = info.files().file_size(file_idx)
|
| 112 |
+
f_name = os.path.basename(info.files().file_path(file_idx))
|
| 113 |
|
| 114 |
+
pr_start = info.map_file(file_idx, 0, 1).piece
|
| 115 |
+
pr_end = info.map_file(file_idx, f_size - 1, 1).piece
|
| 116 |
+
total_pieces = pr_end - pr_start + 1
|
| 117 |
+
pieces_per_shard = total_pieces // total
|
| 118 |
+
|
| 119 |
+
my_start_piece = pr_start + (part - 1) * pieces_per_shard
|
| 120 |
+
my_end_piece = my_start_piece + pieces_per_shard if part < total else pr_end + 1
|
| 121 |
+
|
| 122 |
+
print(f"[ENGINE] Shard {part}: Sniping Pieces {my_start_piece} to {my_end_piece-1}", flush=True)
|
| 123 |
+
|
| 124 |
+
for i in range(info.num_pieces()):
|
| 125 |
+
if my_start_piece <= i < my_end_piece: handle.piece_priority(i, 7)
|
| 126 |
+
else: handle.piece_priority(i, 0)
|
| 127 |
+
|
| 128 |
+
current_task_info[u_id] = {"current_file": f_name, "part": part, "total": total, "progress": 0}
|
| 129 |
+
|
| 130 |
+
pieces_wanted = my_end_piece - my_start_piece
|
| 131 |
+
while True:
|
| 132 |
+
s = handle.status()
|
| 133 |
+
pieces_done = sum(1 for i in range(my_start_piece, my_end_piece) if handle.have_piece(i))
|
| 134 |
+
prog = (pieces_done / pieces_wanted) * 100 if pieces_wanted > 0 else 100
|
| 135 |
+
current_task_info[u_id]['progress'] = round(prog, 2)
|
| 136 |
|
| 137 |
+
if pieces_done >= pieces_wanted: break
|
| 138 |
+
time.sleep(5)
|
| 139 |
+
|
| 140 |
+
print(f"[ENGINE] 🚀 Uploading Shard {part}...", flush=True)
|
| 141 |
+
chunk_size_bytes = 500 * 1024 * 1024
|
| 142 |
+
|
| 143 |
+
byte_start = (my_start_piece - pr_start) * info.piece_length()
|
| 144 |
+
byte_end = f_size if part == total else (my_end_piece - pr_start) * info.piece_length()
|
| 145 |
+
starting_chunk_index = byte_start // chunk_size_bytes
|
| 146 |
+
|
| 147 |
+
local_full = os.path.join(work_dir, info.files().file_path(file_idx))
|
| 148 |
+
|
| 149 |
+
current_chunk_idx = starting_chunk_index
|
| 150 |
+
bytes_read = 0
|
| 151 |
+
total_bytes_to_read = byte_end - byte_start
|
| 152 |
+
|
| 153 |
+
with open(local_full, "rb") as f_in:
|
| 154 |
+
f_in.seek(byte_start)
|
| 155 |
+
while bytes_read < total_bytes_to_read:
|
| 156 |
+
to_read = min(chunk_size_bytes, total_bytes_to_read - bytes_read)
|
| 157 |
+
chunk = f_in.read(to_read)
|
| 158 |
+
if not chunk: break
|
| 159 |
+
|
| 160 |
+
api.upload_file(path_or_fileobj=chunk, path_in_repo=f"{u_id}/{current_chunk_idx}.bin", repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
|
| 161 |
+
bytes_read += len(chunk)
|
| 162 |
+
current_chunk_idx += 1
|
| 163 |
+
|
| 164 |
+
meta_data = {"size": f_size, "chunk_size_mb": 500, "total_chunks": (f_size // chunk_size_bytes) + 1}
|
| 165 |
+
with open(f"{work_dir}/meta_part{part}.json", "w") as m: json.dump(meta_data, m)
|
| 166 |
+
api.upload_file(path_or_fileobj=f"{work_dir}/meta_part{part}.json", path_in_repo=f"{u_id}/meta_part{part}.json", repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
|
| 167 |
+
|
| 168 |
+
os.remove(local_full)
|
| 169 |
+
print(f"[ENGINE] ✅ Shard {part} Complete.", flush=True)
|
| 170 |
+
|
| 171 |
+
# ⚡ FIX: BULLETPROOF ERROR LOGGING ⚡
|
| 172 |
+
except Exception as e:
|
| 173 |
+
print(f"\n[CRITICAL ERROR in Shard {part}] {e}\n{traceback.format_exc()}\n", flush=True)
|
| 174 |
finally:
|
| 175 |
if handle: ses.remove_torrent(handle)
|
| 176 |
active_handles.pop(u_id, None); current_task_info.pop(u_id, None); active_vaults.discard(u_id)
|
|
|
|
| 180 |
def archive():
|
| 181 |
u_id, info_hash = request.args.get('id'), request.args.get('hash')
|
| 182 |
part, total = int(request.args.get('part', 1)), int(request.args.get('total', 1))
|
| 183 |
+
|
| 184 |
print(f"[API ROUTE] Received command: Shard {part} of {total} for {u_id}", flush=True)
|
| 185 |
|
| 186 |
if u_id not in active_vaults:
|
| 187 |
active_vaults.add(u_id)
|
| 188 |
+
threading.Thread(target=do_piece_shard, args=(u_id, info_hash, part, total)).start()
|
| 189 |
return "Sniper Dispatched", 200
|
| 190 |
+
|
| 191 |
+
print(f"[API ROUTE] Busy. Ignored.", flush=True)
|
| 192 |
return "Busy", 200
|
| 193 |
|
| 194 |
+
# ==========================================
|
| 195 |
+
# COMBINED STREAM PROXY
|
| 196 |
+
# ==========================================
|
| 197 |
+
@app.route('/stream/<u_id>')
|
| 198 |
+
def stream_vault(u_id):
|
| 199 |
+
auth = {"Authorization": f"Bearer {TOKEN}"}
|
| 200 |
+
meta = None
|
| 201 |
+
for p in [1, 2]:
|
| 202 |
+
try:
|
| 203 |
+
r = requests.get(f"https://huggingface.co/datasets/{REPO_ID}/resolve/main/{u_id}/meta_part{p}.json", headers=auth, timeout=5)
|
| 204 |
+
if r.status_code == 200: meta = r.json(); break
|
| 205 |
+
except: pass
|
| 206 |
+
if not meta: return "File not vaulted yet.", 404
|
| 207 |
+
|
| 208 |
+
total_size, c_size_mb, num_chunks = meta.get('size', 0), meta.get('chunk_size_mb', 500), meta.get('total_chunks', 0)
|
| 209 |
+
range_header = request.headers.get('Range', None)
|
| 210 |
+
byte_start = int(range_header.replace('bytes=', '').split('-')[0]) if range_header else 0
|
| 211 |
+
|
| 212 |
+
def generate():
|
| 213 |
+
c_bytes = c_size_mb * 1024 * 1024
|
| 214 |
+
for i in range(byte_start // c_bytes, num_chunks):
|
| 215 |
+
url = f"https://huggingface.co/datasets/{REPO_ID}/resolve/main/{u_id}/{i}.bin"
|
| 216 |
+
headers = auth.copy()
|
| 217 |
+
if i == (byte_start // c_bytes) and byte_start % c_bytes > 0: headers["Range"] = f"bytes={byte_start % c_bytes}-"
|
| 218 |
+
with requests.get(url, headers=headers, stream=True) as r:
|
| 219 |
+
if r.status_code in [200, 206]:
|
| 220 |
+
for data in r.iter_content(chunk_size=1024*512): yield data
|
| 221 |
+
else: break
|
| 222 |
+
|
| 223 |
+
headers = {"Content-Length": str(total_size - byte_start), "Content-Type": "video/x-matroska", "Accept-Ranges": "bytes"}
|
| 224 |
+
if range_header: headers["Content-Range"] = f"bytes {byte_start}-{total_size-1}/{total_size}"
|
| 225 |
+
return Response(stream_with_context(generate()), status=206 if range_header else 200, headers=headers)
|
| 226 |
|
| 227 |
@app.route('/delete_active/<u_id>', methods=['POST'])
|
| 228 |
def delete_active(u_id):
|