amri12310 commited on
Commit
85faa5d
·
verified ·
1 Parent(s): 52426c3

Update engine.py

Browse files
Files changed (1) hide show
  1. engine.py +105 -58
engine.py CHANGED
@@ -6,7 +6,9 @@ import threading, shutil, json, time, glob, subprocess, urllib.parse, logging, s
6
  import warnings, requests, queue, re
7
  import libtorrent as lt
8
  from flask import Flask, request, Response, stream_with_context, render_template_string, redirect, url_for, abort
9
- from huggingface_hub import HfApi
 
 
10
 
11
  # --- SILENCE WARNINGS ---
12
  warnings.filterwarnings("ignore", category=DeprecationWarning)
@@ -71,7 +73,7 @@ def fetch_cinemeta(u_id):
71
  return meta
72
 
73
  # ==========================================
74
- # FRONT-END DASHBOARD
75
  # ==========================================
76
  DASHBOARD_HTML = """
77
  <!DOCTYPE html>
@@ -153,31 +155,54 @@ DASHBOARD_HTML = """
153
 
154
  <div class="bg-slate-800 rounded-xl p-6 border border-slate-700 shadow-xl">
155
  <h2 class="text-xl font-semibold mb-6 flex items-center text-purple-400">
156
- 📦 Stored Vault (Hugging Face)
157
  </h2>
158
- <div class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-6">
159
- {% for item in vault_folders %}
160
- <div class="bg-slate-900 rounded-xl border border-slate-700 flex overflow-hidden shadow-lg hover:border-slate-500 transition-colors h-32 relative">
161
- <div class="w-20 flex-shrink-0">
162
- <img src="{{ item.meta.poster }}" class="w-full h-full object-cover">
163
- </div>
164
- <div class="p-3 flex flex-col justify-between flex-1">
165
- <div>
166
- <h3 class="text-sm font-bold text-blue-300 leading-tight line-clamp-2">{{ item.meta.title }}</h3>
167
- {% if item.meta.is_series %}
168
- <p class="text-xs text-purple-400 mt-0.5">{{ item.meta.s_e }} {{ '- ' ~ item.meta.ep_name if item.meta.ep_name else '' }}</p>
169
- {% endif %}
170
- <p class="text-[10px] text-slate-500 mt-1 font-mono truncate" title="{{ item.id }}">{{ item.id[:15] }}...</p>
171
  </div>
172
- <div class="flex justify-between items-center mt-2">
173
- <span class="px-2 py-0.5 rounded bg-green-900/30 text-green-400 text-[10px] font-bold border border-green-800">VAULTED</span>
174
- <form action="/delete_vaulted/{{ item.id }}?key={{ key }}" method="POST">
175
- <button class="text-red-500 hover:text-red-400 text-[10px] font-bold hover:underline transition-colors">DELETE</button>
176
- </form>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177
  </div>
178
  </div>
179
  </div>
180
  {% endfor %}
 
 
 
 
 
 
181
  </div>
182
  </div>
183
  </div>
@@ -191,9 +216,22 @@ def dashboard():
191
  try:
192
  files = api.list_repo_files(repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
193
  raw_folders = sorted(list(set([f.split('/')[0] for f in files if '/' in f])))
194
- vaulted_items = [{"id": f, "meta": fetch_cinemeta(f)} for f in raw_folders]
195
- except: vaulted_items = []
196
- return render_template_string(DASHBOARD_HTML, handles=active_handles, vault_folders=vaulted_items, task_info=current_task_info, scout_status=scout_status, key=ADMIN_KEY, addon_url=ADDON_URL)
 
 
 
 
 
 
 
 
 
 
 
 
 
197
 
198
  # ==========================================
199
  # AGGRESSIVE I/O LIBTORRENT SETTINGS
@@ -218,8 +256,6 @@ settings = {
218
  }
219
  ses = lt.session(settings)
220
 
221
-
222
-
223
  @app.route('/inject_peers', methods=['POST'])
224
  def inject_peers():
225
  data = request.get_json()
@@ -283,51 +319,62 @@ def get_status(u_id):
283
  # THE GOLD STANDARD: HYBRID SWARM ARCHITECTURE
284
  # ==========================================
285
  def process_and_upload(filepath, target_u_id, start_chunk, end_chunk, chunk_size, is_movie_shard, part):
286
- """Slices a raw file directly into memory, uploads it instantly, and deletes it to save disk space."""
287
  v_size = os.path.getsize(filepath)
288
  temp_up = f"/app/upload/{target_u_id}"
289
  os.makedirs(temp_up, exist_ok=True)
290
 
 
 
 
291
  with open(filepath, 'rb') as f:
292
  f.seek(start_chunk * chunk_size)
293
- for i in range(start_chunk, end_chunk + 1):
294
- data = f.read(chunk_size)
295
- if not data: break
 
296
 
297
- chunk_path = f"{temp_up}/{i}.bin"
298
- with open(chunk_path, "wb") as out:
299
- out.write(data)
 
 
300
 
301
- # ⚡ PAC-MAN UPLOADER: Upload the chunk instantly and delete it from the hard drive
302
- try:
303
- api.upload_file(
304
- path_or_fileobj=chunk_path,
305
- path_in_repo=f"{target_u_id}/{i}.bin",
306
- repo_id=REPO_ID,
307
- repo_type="dataset",
308
- token=TOKEN
309
- )
310
- except Exception as e:
311
- print(f"[ERROR] Failed to upload chunk {i}: {e}", flush=True)
312
 
313
- # Free up the 500 MB instantly!
314
- try: os.remove(chunk_path)
315
- except: pass
 
316
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
317
  if not is_movie_shard or part == 1:
318
  meta_path = f"{temp_up}/meta.json"
319
- with open(meta_path, "w") as f:
320
- total_chunks = (v_size // chunk_size) + (1 if v_size % chunk_size > 0 else 0)
321
- json.dump({"size": v_size, "chunks": total_chunks, "chunk_size_mb": int(chunk_size / 1024 / 1024)}, f)
322
-
323
  try:
324
- api.upload_file(
325
- path_or_fileobj=meta_path,
326
- path_in_repo=f"{target_u_id}/meta.json",
327
- repo_id=REPO_ID,
328
- repo_type="dataset",
329
- token=TOKEN
330
- )
331
  except: pass
332
 
333
  shutil.rmtree(temp_up, ignore_errors=True)
 
6
  import warnings, requests, queue, re
7
  import libtorrent as lt
8
  from flask import Flask, request, Response, stream_with_context, render_template_string, redirect, url_for, abort
9
+
10
+ # ⚡ ADDED CommitOperationAdd FOR THE BATCH-OF-5 UPLOADER
11
+ from huggingface_hub import HfApi, CommitOperationAdd
12
 
13
  # --- SILENCE WARNINGS ---
14
  warnings.filterwarnings("ignore", category=DeprecationWarning)
 
73
  return meta
74
 
75
  # ==========================================
76
+ # FRONT-END DASHBOARD (SMART FOLDERS)
77
  # ==========================================
78
  DASHBOARD_HTML = """
79
  <!DOCTYPE html>
 
155
 
156
  <div class="bg-slate-800 rounded-xl p-6 border border-slate-700 shadow-xl">
157
  <h2 class="text-xl font-semibold mb-6 flex items-center text-purple-400">
158
+ 📦 Stored Vault (Organized Folders)
159
  </h2>
160
+
161
+ <div class="space-y-4">
162
+ {% for imdb, show in grouped_vault.items() %}
163
+ <div class="bg-slate-900 rounded-xl border border-slate-700 overflow-hidden shadow-lg">
164
+ <div class="p-4 bg-slate-800 border-b border-slate-700 flex justify-between items-center cursor-pointer hover:bg-slate-700 transition" onclick="document.getElementById('group-{{ imdb }}').classList.toggle('hidden')">
165
+ <div class="flex items-center gap-4">
166
+ <img src="{{ show.poster }}" class="w-12 h-16 object-cover rounded shadow-md border border-slate-600">
167
+ <div>
168
+ <h3 class="text-lg font-bold text-blue-300">{{ show.title }}</h3>
169
+ <p class="text-xs text-slate-400 mt-1 font-mono">📁 {{ show.episodes|length }} Vaulted Items</p>
170
+ </div>
 
 
171
  </div>
172
+ <span class="text-slate-500 text-sm font-bold">CLICK TO EXPAND ▼</span>
173
+ </div>
174
+
175
+ <div id="group-{{ imdb }}" class="hidden p-4 bg-slate-900/50">
176
+ <div class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
177
+ {% for item in show.episodes %}
178
+ <div class="bg-slate-800 rounded-lg border border-slate-700 p-3 flex flex-col justify-between hover:border-slate-500 transition-colors">
179
+ <div>
180
+ {% if item.meta.is_series %}
181
+ <h4 class="text-sm font-bold text-purple-400">{{ item.meta.s_e }}</h4>
182
+ <p class="text-xs text-slate-300 truncate">{{ item.meta.ep_name }}</p>
183
+ {% else %}
184
+ <h4 class="text-sm font-bold text-blue-300">Movie File</h4>
185
+ {% endif %}
186
+ <p class="text-[10px] text-slate-500 mt-1 font-mono truncate" title="{{ item.id }}">{{ item.id[:20] }}...</p>
187
+ </div>
188
+ <div class="flex justify-between items-center mt-3 pt-3 border-t border-slate-700">
189
+ <span class="px-2 py-0.5 rounded bg-green-900/30 text-green-400 text-[10px] font-bold border border-green-800">VAULTED</span>
190
+ <form action="/delete_vaulted/{{ item.id }}?key={{ key }}" method="POST">
191
+ <button class="text-red-500 hover:text-red-400 text-[10px] font-bold hover:underline transition-colors">DELETE</button>
192
+ </form>
193
+ </div>
194
+ </div>
195
+ {% endfor %}
196
  </div>
197
  </div>
198
  </div>
199
  {% endfor %}
200
+
201
+ {% if not grouped_vault %}
202
+ <div class="py-12 text-center text-slate-500 italic font-mono bg-slate-900/50 rounded-xl border border-dashed border-slate-700">
203
+ --- Vault is Empty ---
204
+ </div>
205
+ {% endif %}
206
  </div>
207
  </div>
208
  </div>
 
216
  try:
217
  files = api.list_repo_files(repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
218
  raw_folders = sorted(list(set([f.split('/')[0] for f in files if '/' in f])))
219
+
220
+ # ⚡ SMART GROUPING LOGIC FOR THE UI
221
+ grouped_vault = {}
222
+ for f in raw_folders:
223
+ meta = fetch_cinemeta(f)
224
+ imdb_id = f.split('_')[0]
225
+ if imdb_id not in grouped_vault:
226
+ grouped_vault[imdb_id] = {
227
+ "title": meta['title'],
228
+ "poster": meta['poster'],
229
+ "episodes": []
230
+ }
231
+ grouped_vault[imdb_id]["episodes"].append({"id": f, "meta": meta})
232
+
233
+ except: grouped_vault = {}
234
+ return render_template_string(DASHBOARD_HTML, handles=active_handles, grouped_vault=grouped_vault, task_info=current_task_info, scout_status=scout_status, key=ADMIN_KEY, addon_url=ADDON_URL)
235
 
236
  # ==========================================
237
  # AGGRESSIVE I/O LIBTORRENT SETTINGS
 
256
  }
257
  ses = lt.session(settings)
258
 
 
 
259
  @app.route('/inject_peers', methods=['POST'])
260
  def inject_peers():
261
  data = request.get_json()
 
319
  # THE GOLD STANDARD: HYBRID SWARM ARCHITECTURE
320
  # ==========================================
321
  def process_and_upload(filepath, target_u_id, start_chunk, end_chunk, chunk_size, is_movie_shard, part):
322
+ """⚡ BATCH-OF-5 UPLOADER: Bypasses the Hugging Face commit limit and prevents 50GB crashes."""
323
  v_size = os.path.getsize(filepath)
324
  temp_up = f"/app/upload/{target_u_id}"
325
  os.makedirs(temp_up, exist_ok=True)
326
 
327
+ batch_size = 5 # Commits 5 chunks at once (Reduces API calls by 500%)
328
+ current_idx = start_chunk
329
+
330
  with open(filepath, 'rb') as f:
331
  f.seek(start_chunk * chunk_size)
332
+
333
+ while current_idx <= end_chunk:
334
+ ops = []
335
+ chunks_in_batch = []
336
 
337
+ # Slice 5 chunks to the hard drive
338
+ for _ in range(batch_size):
339
+ if current_idx > end_chunk: break
340
+ data = f.read(chunk_size)
341
+ if not data: break
342
 
343
+ c_path = f"{temp_up}/{current_idx}.bin"
344
+ with open(c_path, "wb") as out:
345
+ out.write(data)
 
 
 
 
 
 
 
 
346
 
347
+ # Prepare the batch operation for Hugging Face
348
+ ops.append(CommitOperationAdd(path_in_repo=f"{target_u_id}/{current_idx}.bin", path_or_fileobj=c_path))
349
+ chunks_in_batch.append(c_path)
350
+ current_idx += 1
351
 
352
+ # Fire the single batched API commit
353
+ if ops:
354
+ try:
355
+ api.create_commit(
356
+ repo_id=REPO_ID,
357
+ repo_type="dataset",
358
+ operations=ops,
359
+ commit_message=f"Vaulting {target_u_id} batch",
360
+ token=TOKEN
361
+ )
362
+ except Exception as e:
363
+ print(f"[API ERROR] Batch commit failed: {e}", flush=True)
364
+
365
+ # Instantly delete the 5 chunks from disk to prevent the 50GB crash
366
+ for p in chunks_in_batch:
367
+ try: os.remove(p)
368
+ except: pass
369
+
370
+ # Upload the metadata file at the end
371
  if not is_movie_shard or part == 1:
372
  meta_path = f"{temp_up}/meta.json"
373
+ total_chunks = (v_size // chunk_size) + (1 if v_size % chunk_size > 0 else 0)
374
+ with open(meta_path, "w") as m:
375
+ json.dump({"size": v_size, "chunks": total_chunks, "chunk_size_mb": int(chunk_size / 1024 / 1024)}, m)
 
376
  try:
377
+ api.upload_file(path_or_fileobj=meta_path, path_in_repo=f"{target_u_id}/meta.json", repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
 
 
 
 
 
 
378
  except: pass
379
 
380
  shutil.rmtree(temp_up, ignore_errors=True)