Spaces:
Paused
Paused
Update engine.py
Browse files
engine.py
CHANGED
|
@@ -6,7 +6,9 @@ import threading, shutil, json, time, glob, subprocess, urllib.parse, logging, s
|
|
| 6 |
import warnings, requests, queue, re
|
| 7 |
import libtorrent as lt
|
| 8 |
from flask import Flask, request, Response, stream_with_context, render_template_string, redirect, url_for, abort
|
| 9 |
-
|
|
|
|
|
|
|
| 10 |
|
| 11 |
# --- SILENCE WARNINGS ---
|
| 12 |
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
|
@@ -71,7 +73,7 @@ def fetch_cinemeta(u_id):
|
|
| 71 |
return meta
|
| 72 |
|
| 73 |
# ==========================================
|
| 74 |
-
# FRONT-END DASHBOARD
|
| 75 |
# ==========================================
|
| 76 |
DASHBOARD_HTML = """
|
| 77 |
<!DOCTYPE html>
|
|
@@ -153,31 +155,54 @@ DASHBOARD_HTML = """
|
|
| 153 |
|
| 154 |
<div class="bg-slate-800 rounded-xl p-6 border border-slate-700 shadow-xl">
|
| 155 |
<h2 class="text-xl font-semibold mb-6 flex items-center text-purple-400">
|
| 156 |
-
📦 Stored Vault (
|
| 157 |
</h2>
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
<
|
| 169 |
-
{% endif %}
|
| 170 |
-
<p class="text-[10px] text-slate-500 mt-1 font-mono truncate" title="{{ item.id }}">{{ item.id[:15] }}...</p>
|
| 171 |
</div>
|
| 172 |
-
<
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 177 |
</div>
|
| 178 |
</div>
|
| 179 |
</div>
|
| 180 |
{% endfor %}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 181 |
</div>
|
| 182 |
</div>
|
| 183 |
</div>
|
|
@@ -191,9 +216,22 @@ def dashboard():
|
|
| 191 |
try:
|
| 192 |
files = api.list_repo_files(repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
|
| 193 |
raw_folders = sorted(list(set([f.split('/')[0] for f in files if '/' in f])))
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 197 |
|
| 198 |
# ==========================================
|
| 199 |
# AGGRESSIVE I/O LIBTORRENT SETTINGS
|
|
@@ -218,8 +256,6 @@ settings = {
|
|
| 218 |
}
|
| 219 |
ses = lt.session(settings)
|
| 220 |
|
| 221 |
-
|
| 222 |
-
|
| 223 |
@app.route('/inject_peers', methods=['POST'])
|
| 224 |
def inject_peers():
|
| 225 |
data = request.get_json()
|
|
@@ -283,51 +319,62 @@ def get_status(u_id):
|
|
| 283 |
# THE GOLD STANDARD: HYBRID SWARM ARCHITECTURE
|
| 284 |
# ==========================================
|
| 285 |
def process_and_upload(filepath, target_u_id, start_chunk, end_chunk, chunk_size, is_movie_shard, part):
|
| 286 |
-
"""
|
| 287 |
v_size = os.path.getsize(filepath)
|
| 288 |
temp_up = f"/app/upload/{target_u_id}"
|
| 289 |
os.makedirs(temp_up, exist_ok=True)
|
| 290 |
|
|
|
|
|
|
|
|
|
|
| 291 |
with open(filepath, 'rb') as f:
|
| 292 |
f.seek(start_chunk * chunk_size)
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
|
|
|
| 296 |
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
|
|
|
|
|
|
| 300 |
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
path_or_fileobj=chunk_path,
|
| 305 |
-
path_in_repo=f"{target_u_id}/{i}.bin",
|
| 306 |
-
repo_id=REPO_ID,
|
| 307 |
-
repo_type="dataset",
|
| 308 |
-
token=TOKEN
|
| 309 |
-
)
|
| 310 |
-
except Exception as e:
|
| 311 |
-
print(f"[ERROR] Failed to upload chunk {i}: {e}", flush=True)
|
| 312 |
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
|
|
|
| 316 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 317 |
if not is_movie_shard or part == 1:
|
| 318 |
meta_path = f"{temp_up}/meta.json"
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
json.dump({"size": v_size, "chunks": total_chunks, "chunk_size_mb": int(chunk_size / 1024 / 1024)},
|
| 322 |
-
|
| 323 |
try:
|
| 324 |
-
api.upload_file(
|
| 325 |
-
path_or_fileobj=meta_path,
|
| 326 |
-
path_in_repo=f"{target_u_id}/meta.json",
|
| 327 |
-
repo_id=REPO_ID,
|
| 328 |
-
repo_type="dataset",
|
| 329 |
-
token=TOKEN
|
| 330 |
-
)
|
| 331 |
except: pass
|
| 332 |
|
| 333 |
shutil.rmtree(temp_up, ignore_errors=True)
|
|
|
|
| 6 |
import warnings, requests, queue, re
|
| 7 |
import libtorrent as lt
|
| 8 |
from flask import Flask, request, Response, stream_with_context, render_template_string, redirect, url_for, abort
|
| 9 |
+
|
| 10 |
+
# ⚡ ADDED CommitOperationAdd FOR THE BATCH-OF-5 UPLOADER
|
| 11 |
+
from huggingface_hub import HfApi, CommitOperationAdd
|
| 12 |
|
| 13 |
# --- SILENCE WARNINGS ---
|
| 14 |
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
|
|
|
| 73 |
return meta
|
| 74 |
|
| 75 |
# ==========================================
|
| 76 |
+
# FRONT-END DASHBOARD (SMART FOLDERS)
|
| 77 |
# ==========================================
|
| 78 |
DASHBOARD_HTML = """
|
| 79 |
<!DOCTYPE html>
|
|
|
|
| 155 |
|
| 156 |
<div class="bg-slate-800 rounded-xl p-6 border border-slate-700 shadow-xl">
|
| 157 |
<h2 class="text-xl font-semibold mb-6 flex items-center text-purple-400">
|
| 158 |
+
📦 Stored Vault (Organized Folders)
|
| 159 |
</h2>
|
| 160 |
+
|
| 161 |
+
<div class="space-y-4">
|
| 162 |
+
{% for imdb, show in grouped_vault.items() %}
|
| 163 |
+
<div class="bg-slate-900 rounded-xl border border-slate-700 overflow-hidden shadow-lg">
|
| 164 |
+
<div class="p-4 bg-slate-800 border-b border-slate-700 flex justify-between items-center cursor-pointer hover:bg-slate-700 transition" onclick="document.getElementById('group-{{ imdb }}').classList.toggle('hidden')">
|
| 165 |
+
<div class="flex items-center gap-4">
|
| 166 |
+
<img src="{{ show.poster }}" class="w-12 h-16 object-cover rounded shadow-md border border-slate-600">
|
| 167 |
+
<div>
|
| 168 |
+
<h3 class="text-lg font-bold text-blue-300">{{ show.title }}</h3>
|
| 169 |
+
<p class="text-xs text-slate-400 mt-1 font-mono">📁 {{ show.episodes|length }} Vaulted Items</p>
|
| 170 |
+
</div>
|
|
|
|
|
|
|
| 171 |
</div>
|
| 172 |
+
<span class="text-slate-500 text-sm font-bold">CLICK TO EXPAND ▼</span>
|
| 173 |
+
</div>
|
| 174 |
+
|
| 175 |
+
<div id="group-{{ imdb }}" class="hidden p-4 bg-slate-900/50">
|
| 176 |
+
<div class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
|
| 177 |
+
{% for item in show.episodes %}
|
| 178 |
+
<div class="bg-slate-800 rounded-lg border border-slate-700 p-3 flex flex-col justify-between hover:border-slate-500 transition-colors">
|
| 179 |
+
<div>
|
| 180 |
+
{% if item.meta.is_series %}
|
| 181 |
+
<h4 class="text-sm font-bold text-purple-400">{{ item.meta.s_e }}</h4>
|
| 182 |
+
<p class="text-xs text-slate-300 truncate">{{ item.meta.ep_name }}</p>
|
| 183 |
+
{% else %}
|
| 184 |
+
<h4 class="text-sm font-bold text-blue-300">Movie File</h4>
|
| 185 |
+
{% endif %}
|
| 186 |
+
<p class="text-[10px] text-slate-500 mt-1 font-mono truncate" title="{{ item.id }}">{{ item.id[:20] }}...</p>
|
| 187 |
+
</div>
|
| 188 |
+
<div class="flex justify-between items-center mt-3 pt-3 border-t border-slate-700">
|
| 189 |
+
<span class="px-2 py-0.5 rounded bg-green-900/30 text-green-400 text-[10px] font-bold border border-green-800">VAULTED</span>
|
| 190 |
+
<form action="/delete_vaulted/{{ item.id }}?key={{ key }}" method="POST">
|
| 191 |
+
<button class="text-red-500 hover:text-red-400 text-[10px] font-bold hover:underline transition-colors">DELETE</button>
|
| 192 |
+
</form>
|
| 193 |
+
</div>
|
| 194 |
+
</div>
|
| 195 |
+
{% endfor %}
|
| 196 |
</div>
|
| 197 |
</div>
|
| 198 |
</div>
|
| 199 |
{% endfor %}
|
| 200 |
+
|
| 201 |
+
{% if not grouped_vault %}
|
| 202 |
+
<div class="py-12 text-center text-slate-500 italic font-mono bg-slate-900/50 rounded-xl border border-dashed border-slate-700">
|
| 203 |
+
--- Vault is Empty ---
|
| 204 |
+
</div>
|
| 205 |
+
{% endif %}
|
| 206 |
</div>
|
| 207 |
</div>
|
| 208 |
</div>
|
|
|
|
| 216 |
try:
|
| 217 |
files = api.list_repo_files(repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
|
| 218 |
raw_folders = sorted(list(set([f.split('/')[0] for f in files if '/' in f])))
|
| 219 |
+
|
| 220 |
+
# ⚡ SMART GROUPING LOGIC FOR THE UI
|
| 221 |
+
grouped_vault = {}
|
| 222 |
+
for f in raw_folders:
|
| 223 |
+
meta = fetch_cinemeta(f)
|
| 224 |
+
imdb_id = f.split('_')[0]
|
| 225 |
+
if imdb_id not in grouped_vault:
|
| 226 |
+
grouped_vault[imdb_id] = {
|
| 227 |
+
"title": meta['title'],
|
| 228 |
+
"poster": meta['poster'],
|
| 229 |
+
"episodes": []
|
| 230 |
+
}
|
| 231 |
+
grouped_vault[imdb_id]["episodes"].append({"id": f, "meta": meta})
|
| 232 |
+
|
| 233 |
+
except: grouped_vault = {}
|
| 234 |
+
return render_template_string(DASHBOARD_HTML, handles=active_handles, grouped_vault=grouped_vault, task_info=current_task_info, scout_status=scout_status, key=ADMIN_KEY, addon_url=ADDON_URL)
|
| 235 |
|
| 236 |
# ==========================================
|
| 237 |
# AGGRESSIVE I/O LIBTORRENT SETTINGS
|
|
|
|
| 256 |
}
|
| 257 |
ses = lt.session(settings)
|
| 258 |
|
|
|
|
|
|
|
| 259 |
@app.route('/inject_peers', methods=['POST'])
|
| 260 |
def inject_peers():
|
| 261 |
data = request.get_json()
|
|
|
|
| 319 |
# THE GOLD STANDARD: HYBRID SWARM ARCHITECTURE
|
| 320 |
# ==========================================
|
| 321 |
def process_and_upload(filepath, target_u_id, start_chunk, end_chunk, chunk_size, is_movie_shard, part):
|
| 322 |
+
"""⚡ BATCH-OF-5 UPLOADER: Bypasses the Hugging Face commit limit and prevents 50GB crashes."""
|
| 323 |
v_size = os.path.getsize(filepath)
|
| 324 |
temp_up = f"/app/upload/{target_u_id}"
|
| 325 |
os.makedirs(temp_up, exist_ok=True)
|
| 326 |
|
| 327 |
+
batch_size = 5 # Commits 5 chunks at once (Reduces API calls by 500%)
|
| 328 |
+
current_idx = start_chunk
|
| 329 |
+
|
| 330 |
with open(filepath, 'rb') as f:
|
| 331 |
f.seek(start_chunk * chunk_size)
|
| 332 |
+
|
| 333 |
+
while current_idx <= end_chunk:
|
| 334 |
+
ops = []
|
| 335 |
+
chunks_in_batch = []
|
| 336 |
|
| 337 |
+
# Slice 5 chunks to the hard drive
|
| 338 |
+
for _ in range(batch_size):
|
| 339 |
+
if current_idx > end_chunk: break
|
| 340 |
+
data = f.read(chunk_size)
|
| 341 |
+
if not data: break
|
| 342 |
|
| 343 |
+
c_path = f"{temp_up}/{current_idx}.bin"
|
| 344 |
+
with open(c_path, "wb") as out:
|
| 345 |
+
out.write(data)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 346 |
|
| 347 |
+
# Prepare the batch operation for Hugging Face
|
| 348 |
+
ops.append(CommitOperationAdd(path_in_repo=f"{target_u_id}/{current_idx}.bin", path_or_fileobj=c_path))
|
| 349 |
+
chunks_in_batch.append(c_path)
|
| 350 |
+
current_idx += 1
|
| 351 |
|
| 352 |
+
# Fire the single batched API commit
|
| 353 |
+
if ops:
|
| 354 |
+
try:
|
| 355 |
+
api.create_commit(
|
| 356 |
+
repo_id=REPO_ID,
|
| 357 |
+
repo_type="dataset",
|
| 358 |
+
operations=ops,
|
| 359 |
+
commit_message=f"Vaulting {target_u_id} batch",
|
| 360 |
+
token=TOKEN
|
| 361 |
+
)
|
| 362 |
+
except Exception as e:
|
| 363 |
+
print(f"[API ERROR] Batch commit failed: {e}", flush=True)
|
| 364 |
+
|
| 365 |
+
# Instantly delete the 5 chunks from disk to prevent the 50GB crash
|
| 366 |
+
for p in chunks_in_batch:
|
| 367 |
+
try: os.remove(p)
|
| 368 |
+
except: pass
|
| 369 |
+
|
| 370 |
+
# Upload the metadata file at the end
|
| 371 |
if not is_movie_shard or part == 1:
|
| 372 |
meta_path = f"{temp_up}/meta.json"
|
| 373 |
+
total_chunks = (v_size // chunk_size) + (1 if v_size % chunk_size > 0 else 0)
|
| 374 |
+
with open(meta_path, "w") as m:
|
| 375 |
+
json.dump({"size": v_size, "chunks": total_chunks, "chunk_size_mb": int(chunk_size / 1024 / 1024)}, m)
|
|
|
|
| 376 |
try:
|
| 377 |
+
api.upload_file(path_or_fileobj=meta_path, path_in_repo=f"{target_u_id}/meta.json", repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 378 |
except: pass
|
| 379 |
|
| 380 |
shutil.rmtree(temp_up, ignore_errors=True)
|