sys-c-engine / engine.py
amri12310's picture
Update engine.py
dc9e015 verified
Raw History Blame Contribute Delete
46.6 kB
# ⚑ CRITICAL: Force Rust Engine BEFORE importing Hugging Face
import os
os.environ["HF_XET_HIGH_PERFORMANCE"] = "1"
import urllib3
import threading, shutil, json, time, glob, subprocess, urllib.parse, logging, struct, socket
import warnings, requests, queue, re
import concurrent.futures # ⚑ ADDED: For Cross-Engine Dashboard Polling
import libtorrent as lt
from flask import Flask, request, Response, stream_with_context, render_template_string, redirect, url_for, abort, jsonify
# ⚑ ADDED CommitOperationDelete FOR THE 1-CLICK WIPERS
from huggingface_hub import HfApi, CommitOperationAdd, CommitOperationDelete
# --- SILENCE WARNINGS ---
warnings.filterwarnings("ignore", category=DeprecationWarning)
log = logging.getLogger('werkzeug')
log.setLevel(logging.ERROR)
app = Flask(__name__)
api = HfApi()
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
stream_session = requests.Session()
adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100, max_retries=3)
stream_session.mount('https://', adapter)
# --- CONFIGURATION ---
REPO_ID = "amri12310/hf-vault-99281-xk9-p02-mlq8-r4x-v9928-zb-v2-alpha-sync"
TOKEN = os.environ.get("HF_TOKEN")
GH_TOKEN = os.environ.get("GH_PAT")
ADMIN_KEY = os.environ.get("ADMIN_KEY")
ADDON_URL = "https://amri12310-sys-b-addon.hf.space"
SYS_E_URL = "https://amri123410-sys-e-engine.hf.space/"
# ⚑ ADDED: ENGINE URLS FOR UNIFIED DASHBOARD MATH
ENGINE_URLS = [
"https://amri12310-sys-b-engine.hf.space",
"https://amri123510-sys-c-engine.hf.space",
"https://amri123410-sys-d-engine.hf.space"
]
active_vaults = set()
completed_vaults = set() # ⚑ ADDED MEMORY BANK FOR FINISHED CHUNKS
active_handles = {}
current_task_info = {}
scout_status = {}
cinemeta_cache = {}
def check_auth():
if request.args.get('key') != ADMIN_KEY: abort(403)
# ==========================================
# METADATA FETCHER (Stremio Cinemeta)
# ==========================================
def fetch_cinemeta(u_id):
if u_id in cinemeta_cache: return cinemeta_cache[u_id]
parts = u_id.split('_')
imdb_id = parts[0]
is_series = len(parts) > 2 and parts[1].startswith('S')
meta = {
"title": imdb_id,
"poster": "https://via.placeholder.com/150x225/1e293b/94a3b8?text=Downloading...",
"ep_name": "",
"is_series": is_series,
"s_e": f"{parts[1]}{parts[2]}" if is_series else ""
}
try:
m_type = "series" if is_series else "movie"
r = requests.get(f"https://v3-cinemeta.strem.io/meta/{m_type}/{imdb_id}.json", timeout=3)
if r.status_code == 200:
data = r.json().get("meta") or {}
if data:
meta["title"] = data.get("name", imdb_id)
meta["poster"] = data.get("poster", meta["poster"])
if is_series:
s, e = int(parts[1][1:]), int(parts[2][1:])
for vid in data.get("videos", []):
if vid.get("season") == s and vid.get("episode") == e:
meta["ep_name"] = vid.get("name", "")
break
except: pass
# ⚑ FIX: Removed HTML escape to fix ' apostrophe bug
meta["title"] = str(meta["title"]).replace('<', '').replace('>', '').replace('&', '&amp;')
meta["ep_name"] = str(meta["ep_name"]).replace('<', '').replace('>', '').replace('&', '&amp;')
cinemeta_cache[u_id] = meta
return meta
def format_eta(seconds):
if seconds >= 86400: return "> 1 Day"
h, m, s = seconds // 3600, (seconds % 3600) // 60, seconds % 60
if h > 0: return f"{h}h {m}m"
if m > 0: return f"{m}m {s}s"
return f"{s}s"
# ⚑ ADDED: SIZE FORMATTER
def format_size(size_bytes):
if size_bytes >= 1024**3: return f"{size_bytes / 1024**3:.2f} GB"
if size_bytes >= 1024**2: return f"{size_bytes / 1024**2:.2f} MB"
return f"{size_bytes / 1024:.2f} KB"
# ==========================================
# FRONT-END DASHBOARDS (SEPARATED)
# ==========================================
NAV_BAR = """
<div class="flex justify-between items-center mb-8">
<h1 class="text-3xl font-bold text-blue-400">πŸš€ Gold Standard Engine</h1>
<div class="flex gap-4">
<a href="/?key={{ key }}" class="px-4 py-2 bg-slate-800 hover:bg-blue-600 rounded text-sm font-bold transition">Live Swarm</a>
<a href="/vault?key={{ key }}" class="px-4 py-2 bg-slate-800 hover:bg-purple-600 rounded text-sm font-bold transition">My Vault</a>
</div>
</div>
"""
ACTIVE_HTML = """
<!DOCTYPE html>
<html>
<head>
<title>Engine | Active Swarm</title>
<script src="https://cdn.tailwindcss.com"></script>
<meta http-equiv="refresh" content="10">
</head>
<body class="bg-slate-900 text-gray-100 font-sans p-8">
<div class="max-w-6xl mx-auto">
""" + NAV_BAR + """
<div class="bg-slate-800 rounded-xl p-6 mb-8 border border-slate-700 shadow-xl">
<h2 class="text-xl font-semibold mb-6 flex items-center text-green-400">
<span class="w-3 h-3 bg-green-500 rounded-full mr-2 animate-pulse"></span>
Active Swarm (Cluster Aggregated)
</h2>
<div class="space-y-4">
{% for uid, c_data in cluster_data.items() %}
<div class="bg-slate-900 p-4 rounded-xl border border-slate-700 flex gap-5 shadow-lg relative overflow-hidden">
<div class="flex-shrink-0 w-24 h-36 z-10">
<img src="{{ c_data.meta.poster }}" class="w-full h-full object-cover rounded shadow-md border border-slate-600">
</div>
<div class="flex-1 flex flex-col justify-between z-10">
<div>
<div class="flex justify-between items-start mb-2">
<h3 class="text-xl font-bold text-blue-300 leading-tight drop-shadow-md">
{{ c_data.meta.title }}
{% if c_data.meta.is_series %}
<span class="text-sm text-purple-400 font-normal ml-2 bg-purple-900/30 px-2 py-0.5 rounded">{{ c_data.meta.s_e }} {{ '- ' ~ c_data.meta.ep_name if c_data.meta.ep_name else '' }}</span>
{% endif %}
</h3>
<form action="{{ addon_url }}/kill/{{ uid }}?key={{ key }}" method="POST">
<button class="bg-red-900/40 border border-red-800 text-red-400 hover:bg-red-600 hover:text-white px-3 py-1 rounded text-[10px] uppercase font-bold transition-colors shadow">KILL TASK (GLOBAL)</button>
</form>
</div>
<div class="grid grid-cols-3 gap-2 text-xs text-slate-300 font-mono bg-slate-950 p-2 rounded border border-slate-800 mb-3">
<div><span class="text-slate-500">Speed:</span> <span class="text-green-400 font-bold">{{ c_data.speed_mb }} MB/s</span></div>
<div class="text-center"><span class="text-slate-500">Size:</span> {{ c_data.done_gb }} / {{ c_data.total_gb }} GB</div>
<div class="text-right"><span class="text-slate-500">ETA:</span> {{ c_data.eta_str }}</div>
</div>
<div class="text-[11px] font-mono text-slate-400 space-y-1 mb-3 bg-slate-900 p-2 rounded">
{% for node in c_data.nodes %}
<div>β€’ Engine {{ node.name }}: {{ node.progress }}% ({{ node.done_gb }}/{{ node.total_gb }} GB) @ {{ node.speed_mb }} MB/s | ETA: {{ node.eta }} | Seeds: {{ node.seeds }} | Peers: {{ node.peers }}</div>
{% endfor %}
</div>
</div>
<div class="w-full bg-slate-800 rounded-full h-3 mb-1 shadow-inner border border-slate-700">
<div class="bg-gradient-to-r from-blue-600 to-blue-400 h-full rounded-full transition-all duration-500" style="width: {{ c_data.progress }}%"></div>
</div>
</div>
</div>
{% endfor %}
{% if not cluster_data %}
<div class="py-12 text-center text-slate-500 italic font-mono bg-slate-900/50 rounded-xl border border-dashed border-slate-700">
--- Swarm Idle. Awaiting Stremio Command ---
</div>
{% endif %}
</div>
</div>
</div>
</body>
</html>
"""
# ⚑ UPDATED: VAULT_HTML NO LONGER AUTOREFRESHES & HAS HASH BADGE
VAULT_HTML = """
<!DOCTYPE html>
<html>
<head>
<title>Engine | Vault</title>
<script src="https://cdn.tailwindcss.com"></script>
</head>
<body class="bg-slate-900 text-gray-100 font-sans p-8">
<div class="max-w-6xl mx-auto">
""" + NAV_BAR + """
<div class="bg-slate-800 rounded-xl p-6 border border-slate-700 shadow-xl">
<h2 class="text-xl font-semibold mb-6 flex items-center text-purple-400">πŸ“¦ Stored Vault (Organized Folders)</h2>
<div class="space-y-4">
{% for imdb, show in grouped_vault.items() %}
<div class="bg-slate-900 rounded-xl border border-slate-700 overflow-hidden shadow-lg">
<div class="p-4 bg-slate-800 border-b border-slate-700 flex justify-between items-center transition hover:bg-slate-700">
<div class="flex items-center gap-4 cursor-pointer flex-1" onclick="toggleVault('{{ imdb }}')">
<img src="{{ show.poster }}" class="w-12 h-16 object-cover rounded shadow-md border border-slate-600">
<div>
<h3 class="text-lg font-bold text-blue-300">
{{ show.title }}
<span class="text-xs text-blue-400 font-mono ml-2 bg-blue-900/50 border border-blue-700/50 px-2 py-0.5 rounded">{{ show.total_size_str }} TOTAL</span>
</h3>
<p class="text-xs text-slate-400 mt-1 font-mono">πŸ“ Contains {{ show.seasons|length }} Group(s)</p>
</div>
</div>
<div class="flex items-center gap-4">
<span class="text-slate-500 text-xs font-bold cursor-pointer hidden md:block" onclick="toggleVault('{{ imdb }}')">EXPAND β–Ό</span>
<form action="/delete_show/{{ imdb }}?key={{ key }}" method="POST" onsubmit="return confirm('WARNING: This will delete ALL seasons and episodes of {{ show.title }}. Are you sure?');">
<button class="bg-red-500/20 text-red-500 hover:bg-red-500 hover:text-white px-3 py-1.5 rounded text-[10px] uppercase font-bold border border-red-700 transition-colors">DELETE SHOW</button>
</form>
</div>
</div>
<div id="group-{{ imdb }}" class="hidden p-4 bg-slate-900/50">
{% for season_name, s_data in show.seasons.items()|sort %}
<div class="mb-6 last:mb-0 border border-slate-700 rounded bg-slate-800/40 p-3 shadow-inner">
<div class="flex justify-between items-center mb-4 border-b border-slate-700 pb-2">
<h4 class="text-md font-bold text-purple-300 flex items-center">
<span class="mr-2">πŸ“Ί</span>
{% if season_name == 'Movie' %}Movie/Standalone{% else %}Season {{ season_name|replace('S', '') }}{% endif %}
<span class="text-xs text-slate-500 font-normal ml-2">({{ s_data.episodes|length }} Items)</span>
<span class="text-xs text-purple-400 font-mono ml-3 bg-purple-900/30 border border-purple-700/50 px-2 py-0.5 rounded">{{ s_data.season_size_str }}</span>
</h4>
{% if season_name != 'Movie' %}
<form action="/delete_season/{{ imdb }}/{{ season_name }}?key={{ key }}" method="POST" onsubmit="return confirm('Are you sure you want to delete ALL episodes in {{ season_name }} of {{ show.title }}?');">
<button class="text-red-500 hover:text-red-400 px-2 py-1 rounded text-[10px] uppercase font-bold border border-red-900/50 hover:border-red-500 transition-colors">DELETE SEASON</button>
</form>
{% endif %}
</div>
<div class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
{% for item in s_data.episodes|sort(attribute='meta.s_e') %}
<div class="bg-slate-800 rounded-lg border border-slate-700 p-3 flex flex-col justify-between hover:border-slate-500 transition-colors shadow">
<div>
{% if item.meta.is_series %}
<h4 class="text-sm font-bold text-blue-200">{{ item.meta.s_e }}</h4>
<p class="text-xs text-slate-400 mt-1 truncate">{{ item.meta.ep_name }}</p>
{% else %}
<h4 class="text-sm font-bold text-blue-200">Movie File</h4>
{% endif %}
<div class="flex items-center gap-2 mt-3">
<span class="text-[10px] text-slate-300 font-mono font-bold bg-slate-700 px-1.5 py-0.5 rounded">{{ item.size_str }}</span>
<span class="text-[10px] text-slate-500 font-mono bg-slate-900 px-1.5 py-0.5 rounded border border-slate-700" title="{{ item.id }}">Hash: <span class="text-blue-400 font-bold">{{ item.id.split('_')[-1][:8] }}</span></span>
</div>
</div>
<div class="flex justify-between items-center mt-3 pt-3 border-t border-slate-700/50">
<a href="{{ sys_e_url }}/stream/{{ item.id }}?dl=1" class="px-2 py-1 bg-blue-600 hover:bg-blue-500 text-white rounded text-[10px] font-bold transition-colors">DOWNLOAD EP</a>
<form action="/delete_vaulted/{{ item.id }}?key={{ key }}" method="POST">
<button class="text-slate-500 hover:text-red-500 text-[10px] font-bold hover:underline transition-colors">DELETE EP</button>
</form>
</div>
</div>
{% endfor %}
</div>
</div>
{% endfor %}
</div>
</div>
{% endfor %}
{% if not grouped_vault %}
<div class="py-12 text-center text-slate-500 italic font-mono bg-slate-900/50 rounded-xl border border-dashed border-slate-700">--- Vault is Empty ---</div>
{% endif %}
</div>
</div>
</div>
<script>
document.addEventListener("DOMContentLoaded", function(event) {
var scrollpos = localStorage.getItem('vault_scrollpos');
if (scrollpos) window.scrollTo(0, scrollpos);
});
window.onbeforeunload = function(e) { localStorage.setItem('vault_scrollpos', window.scrollY); };
function toggleVault(imdbId) {
let el = document.getElementById('group-' + imdbId);
el.classList.toggle('hidden');
if(el.classList.contains('hidden')) localStorage.removeItem('vault_open_' + imdbId);
else localStorage.setItem('vault_open_' + imdbId, 'true');
}
document.addEventListener("DOMContentLoaded", function() {
let groups = document.querySelectorAll('[id^="group-"]');
groups.forEach(function(el) {
let imdbId = el.id.replace('group-', '');
if(localStorage.getItem('vault_open_' + imdbId) === 'true') el.classList.remove('hidden');
});
});
</script>
</body>
</html>
"""
@app.route('/')
def dashboard():
check_auth()
# ⚑ 1. ONLY DO CROSS-ENGINE POLLING HERE
cluster_data = {}
def fetch_engine_status(url, uid):
try: return requests.get(f"{url}/status/{uid}?detailed=1", timeout=7).json()
except: return {"status": "offline"}
for u_id in list(active_vaults):
with concurrent.futures.ThreadPoolExecutor() as executor:
results = list(executor.map(lambda url: fetch_engine_status(url, u_id), ENGINE_URLS))
engines_alive, total_wanted, total_done, total_speed, max_eta = 0, 0, 0, 0, 0
node_list = [] # ⚑ NEW: This holds the individual engine data for the HTML template
for i, r in enumerate(results):
node_name = ["B", "C", "D"][i]
if r.get("status") == "active":
engines_alive += 1
e_w = r.get("wanted_bytes", 0)
e_d = r.get("done_bytes", 0)
e_s = r.get("speed_bytes", 0)
e_eta = r.get("eta_seconds", 0)
total_wanted += e_w
total_done += e_d
total_speed += e_s
max_eta = max(max_eta, e_eta)
# Append individual node stats
node_list.append({
"name": node_name,
"progress": int((e_d / e_w) * 100) if e_w > 0 else 0,
"done_gb": round(e_d / 1024**3, 1),
"total_gb": round(e_w / 1024**3, 1),
"speed_mb": round(e_s / 1024**2, 1),
"eta": format_eta(e_eta),
"seeds": r.get("seeds", "?"),
"peers": r.get("peers", "?")
})
elif r.get("status") == "finished":
node_list.append({
"name": node_name, "progress": 100, "done_gb": "-", "total_gb": "-", "speed_mb": 0.0, "eta": "Done", "seeds": "-", "peers": "-"
})
if engines_alive > 0 or node_list:
cluster_data[u_id] = {
"active_nodes": engines_alive,
"progress": int((total_done / total_wanted) * 100) if total_wanted > 0 else 0,
"total_gb": round(total_wanted / 1024**3, 2),
"done_gb": round(total_done / 1024**3, 2),
"speed_mb": round(total_speed / 1024**2, 2),
"eta_str": format_eta(max_eta),
"meta": fetch_cinemeta(u_id),
"nodes": node_list # ⚑ This injects the list so the HTML can loop it!
}
return render_template_string(ACTIVE_HTML, cluster_data=cluster_data, key=ADMIN_KEY, addon_url=ADDON_URL)
# ⚑ UPDATED: BULK ZERO-SPAM SIZE FETCHER
@app.route('/vault')
def vault_view():
check_auth()
try:
# Grabs metadata for every file in the repo instantly (Zero API Spam)
info = api.dataset_info(repo_id=REPO_ID, files_metadata=True, token=TOKEN)
folder_sizes = {}
for f in info.siblings:
if f.size:
parts = f.rfilename.split('/')
if len(parts) > 1:
folder = parts[0]
folder_sizes[folder] = folder_sizes.get(folder, 0) + f.size
raw_folders = sorted(list(folder_sizes.keys()))
grouped_vault = {}
for f in raw_folders:
meta = fetch_cinemeta(f)
imdb_id = f.split('_')[0]
size_bytes = folder_sizes.get(f, 0)
season_name = "Movie"
if meta['is_series']:
parts = f.split('_')
if len(parts) > 1 and parts[1].startswith('S'):
season_name = parts[1]
if imdb_id not in grouped_vault:
grouped_vault[imdb_id] = {
"title": meta['title'],
"poster": meta['poster'],
"seasons": {},
"total_size_bytes": 0
}
if season_name not in grouped_vault[imdb_id]["seasons"]:
grouped_vault[imdb_id]["seasons"][season_name] = {
"episodes": [],
"season_size_bytes": 0
}
grouped_vault[imdb_id]["seasons"][season_name]["episodes"].append({
"id": f,
"meta": meta,
"size_str": format_size(size_bytes)
})
grouped_vault[imdb_id]["seasons"][season_name]["season_size_bytes"] += size_bytes
grouped_vault[imdb_id]["total_size_bytes"] += size_bytes
for imdb, show in grouped_vault.items():
show["total_size_str"] = format_size(show["total_size_bytes"])
for s_name, s_data in show["seasons"].items():
s_data["season_size_str"] = format_size(s_data["season_size_bytes"])
except Exception as e:
print(f"Vault Load Error: {e}", flush=True)
grouped_vault = {}
return render_template_string(VAULT_HTML, grouped_vault=grouped_vault, key=ADMIN_KEY, sys_e_url=SYS_E_URL)
# ==========================================
# πŸš€ 3.2 GIGABIT (400 MB/s) MASTER CONFIGURATION
# ==========================================
settings = {
'listen_interfaces': '0.0.0.0:6881',
'enable_dht': False,
'dht_bootstrap_nodes': 'router.bittorrent.com:6881,router.utorrent.com:6881,dht.transmissionbt.com:6881',
# ⚑ THE GOLDILOCKS SWARM
'connections_limit': 350,
'unchoke_slots_limit': 150,
# ⚑ TCP PACING UNCAP
'connection_speed': 2000,
'active_downloads': 1,
'aio_threads': 12,
# ⚑ MASSIVE RAM RUNWAYS
'cache_size': 49152, # 2 GB RAM Buffer
'max_queued_disk_bytes': 384 * 1024 * 1024, # 2 GB Disk Write Buffer
# ⚑ TIMEOUTS & ENCRYPTION
'peer_connect_timeout': 3,
'min_reconnect_time': 2,
'request_timeout': 15,
'max_failcount': 1,
'in_enc_policy': 1,
'out_enc_policy': 1,
'strict_end_game_mode': True,
# ⚑ ANNOUNCE PROTOCOLS
'announce_to_all_trackers': True,
'announce_to_all_tiers': True,
'enable_outgoing_utp': False,
'enable_incoming_utp': False,
}
ses = lt.session(settings)
@app.route('/inject_peers', methods=['POST'])
def inject_peers():
data = request.get_json()
u_id, peers = data.get('id'), data.get('peers', [])
handle = active_handles.get(u_id)
if not handle: return "No active handle", 404
count = 0
for p in peers:
try:
handle.connect_peer((p['ip'], int(p['port'])), 0)
count += 1
except: continue
if count > 0: print(f"🎯 [SCOUT] Force-fed {count} peers into {u_id}", flush=True)
return "OK", 200
# ==========================================
# ⚑ HTTP TRACKER PEER INJECTION (FIXED ANTI-DDOS BAN) ⚑
# ==========================================
def force_inject_peers(info_hash, trackers, handle, peer_id):
"""Manually interrogates HTTP trackers using a STATIC peer ID to prevent bans."""
for tracker in trackers:
if not tracker.startswith('http'): continue
try:
params = {
'info_hash': bytes.fromhex(info_hash),
'peer_id': peer_id,
'port': 6881,
'uploaded': 0, 'downloaded': 0, 'left': 999999999,
'compact': 1, 'event': 'started'
}
r = requests.get(tracker, params=params, timeout=5)
peers_index = r.content.find(b'5:peers')
if peers_index == -1: continue
start_idx = r.content.find(b':', peers_index) + 1
peer_data = r.content[start_idx:]
injected_count = 0
for i in range(0, len(peer_data) - 6, 6):
chunk = peer_data[i:i+6]
if len(chunk) < 6: break
ip = socket.inet_ntoa(chunk[:4])
port = struct.unpack(">H", chunk[4:6])[0]
handle.connect_peer((ip, port), 0)
injected_count += 1
if injected_count > 0:
print(f"[INJECT] Forced {injected_count} peers from {tracker}", flush=True)
except Exception:
pass
# ⚑ UPDATED: /status ENDPOINT NOW REPORTS FINISHED STATE TO ADDON
@app.route('/status/<u_id>')
def get_status(u_id):
detailed = request.args.get('detailed') == '1'
if u_id in active_vaults:
if detailed and u_id in active_handles:
s = active_handles[u_id].status()
dl_speed = s.download_rate if s.download_rate > 0 else 1
bytes_left = s.total_wanted - s.total_done
eta_seconds = int(bytes_left / dl_speed)
return jsonify({
"status": "active",
"progress": int(s.progress * 100),
"wanted_bytes": s.total_wanted,
"done_bytes": s.total_done,
"speed_bytes": s.download_rate,
"eta_seconds": eta_seconds,
"seeds": s.num_seeds, # ⚑ ADDED: Connected Seeds
"peers": s.num_peers, # ⚑ ADDED: Connected Peers
"availability": s.distributed_copies # ⚑ ADDED: Swarm Health/Availability
})
return {"status": "active", "progress": 0, "wanted_bytes": 0, "done_bytes": 0, "speed_bytes": 0, "eta_seconds": 9999}, 200
elif u_id in completed_vaults:
return {"status": "finished"}, 200
return {"status": "offline"}, 200
# ==========================================
# ⚑ THE GLOBAL DYNAMIC BUFFER LOGIC WITH DEDUPLICATION ⚑
# ==========================================
def flush_batch(batch_state, base_msg):
"""Fires the API Ping and instantly wipes the Disk Buffer."""
if not batch_state['ops']: return
try:
print(f"πŸ“‘ [BUFFER FLUSH] Committing {len(batch_state['ops'])} items ({batch_state['bytes']/1024**2:.0f} MB)...", flush=True)
api.create_commit(repo_id=REPO_ID, repo_type="dataset", operations=batch_state['ops'], commit_message=base_msg, token=TOKEN)
except Exception as e:
print(f"❌ [API ERROR] {e}", flush=True)
for p in batch_state['paths']:
try: os.remove(p)
except: pass
batch_state['ops'].clear()
batch_state['paths'].clear()
batch_state['bytes'] = 0
batch_state['last_fire'] = time.time()
batch_state['queued_repo_paths'].clear() # ⚑ Reset the Deduplication Lock after successful push
def process_and_queue(filepath, target_u_id, start_chunk, end_chunk, chunk_size, is_movie_shard, part, batch_state):
"""Slices the file directly into the Global Bucket. Commits immediately if it hits 4.5GB."""
DISK_THRESHOLD = 4.5 * 1024 * 1024 * 1024 # ⚑ 4.5GB Wall (Allows safe overshoot to keep chunks perfect)
v_size = os.path.getsize(filepath)
temp_up = f"/app/upload/temp_chunks"
os.makedirs(temp_up, exist_ok=True)
# ⚑ Make sure the deduplication lock exists
if 'queued_repo_paths' not in batch_state:
batch_state['queued_repo_paths'] = set()
current_idx = start_chunk
with open(filepath, 'rb') as f:
f.seek(start_chunk * chunk_size)
while current_idx <= end_chunk:
data = f.read(chunk_size)
if not data: break
repo_path = f"{target_u_id}/{current_idx}.bin"
# ⚑ THE DEDUPLICATION SHIELD: Only add to bucket if we haven't already
if repo_path not in batch_state['queued_repo_paths']:
c_filename = f"{target_u_id}_{current_idx}.bin"
c_path = os.path.join(temp_up, c_filename)
with open(c_path, "wb") as out: out.write(data)
batch_state['ops'].append(CommitOperationAdd(path_in_repo=repo_path, path_or_fileobj=c_path))
batch_state['paths'].append(c_path)
batch_state['bytes'] += len(data)
batch_state['queued_repo_paths'].add(repo_path)
current_idx += 1
# ⚑ DYNAMIC PRESSURE SHIELD: Commit exact mid-file if bucket is full!
if batch_state['bytes'] >= DISK_THRESHOLD:
flush_batch(batch_state, f"Vaulting 4.5GB dynamic chunk batch")
# Add metadata to the buffer when file finishes
if not is_movie_shard or part == 1:
meta_repo_path = f"{target_u_id}/meta.json"
# ⚑ DEDUPLICATION SHIELD FOR META
if meta_repo_path not in batch_state['queued_repo_paths']:
meta_path = os.path.join(temp_up, f"{target_u_id}_meta.json")
total_chunks = (v_size // chunk_size) + (1 if v_size % chunk_size > 0 else 0)
with open(meta_path, "w") as m:
json.dump({"size": v_size, "chunks": total_chunks, "chunk_size_mb": int(chunk_size / 1024 / 1024)}, m)
batch_state['ops'].append(CommitOperationAdd(path_in_repo=meta_repo_path, path_or_fileobj=meta_path))
batch_state['paths'].append(meta_path)
batch_state['queued_repo_paths'].add(meta_repo_path)
# Secondary check for safety
if batch_state['bytes'] >= DISK_THRESHOLD:
flush_batch(batch_state, f"Vaulting 4.5GB dynamic meta batch")
def continuous_peer_pulse(u_id, info_hash, trackers, handle, peer_id):
"""Silently tops off the peer list every 3 minutes until the download finishes."""
while True:
time.sleep(500) # Wait between pulses
try:
# Stop pulsing if the download is finished, killed, or stalled
if u_id not in active_vaults or handle.status().state in [lt.torrent_status.seeding, lt.torrent_status.finished]:
break
print(f"[PULSE] πŸ’‰ 3-Minute mark reached. Topping off peers for {u_id[:8]}...", flush=True)
force_inject_peers(info_hash, trackers, handle, peer_id)
except Exception:
break
def do_archive(u_id, info_hash, part=1, total=1, is_series=False, priority_idx=-1):
handle = None
scout_status[u_id] = "active"
try:
work_dir = f"/app/temp/{u_id}"
os.makedirs(work_dir, exist_ok=True)
current_task_info[u_id] = {"current_file": "Connecting...", "size_gb": "0.0", "meta": fetch_cinemeta(u_id)}
# --- THE MEGA-SCRAPE: HTTP-ONLY TRACKER BOMB ---
print(f"[ENGINE] Fetching ALL live HTTP trackers...", flush=True)
hardcoded_trackers = [
"http://tracker.opentrackr.org:1337/announce",
"https://tracker.tamersunion.org:443/announce",
"http://tracker.dler.org:6969/announce",
"http://tracker1.itzmx.com:8080/announce",
"http://open.acgnxtracker.com:80/announce",
"https://tracker.bt4g.com:443/announce",
"http://vito-tracker.space:6969/announce",
"http://tracker.mywaifu.best:6969/announce",
"https://tracker.lilithraws.org:443/announce",
"http://tracker.renfei.net:8080/announce",
"https://tr.nyacat.pw:443/announce",
"http://tracker.qu.ax:6969/announce",
"http://tracker.waaa.moe:6969/announce",
"https://tracker.nekomi.cn:443/announce"
]
try:
r_trackers = requests.get("https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt", timeout=5)
dynamic_trackers = [t for t in r_trackers.text.split('\n') if t.strip()]
except: dynamic_trackers = []
all_trackers = [t for t in list(set(hardcoded_trackers + dynamic_trackers)) if t.startswith('http')]
tr_string = "".join([f"&tr={urllib.parse.quote(t)}" for t in all_trackers])
magnet = f"magnet:?xt=urn:btih:{info_hash}{tr_string}"
atp = lt.parse_magnet_uri(magnet)
atp.save_path = work_dir
handle = ses.add_torrent(atp)
active_handles[u_id] = handle
# ⚑ GENERATE THE STATIC PEER ID FOR THIS DOWNLOAD ⚑
my_peer_id = b'-SY0001-' + os.urandom(12)
threading.Thread(target=force_inject_peers, args=(info_hash, all_trackers, handle, my_peer_id)).start()
threading.Thread(target=continuous_peer_pulse, args=(u_id, info_hash, all_trackers, handle, my_peer_id), daemon=True).start()
print(f"[ENGINE] Waiting for metadata...", flush=True)
while not handle.status().has_metadata:
time.sleep(1)
if u_id not in active_vaults: raise Exception("Task Terminated")
info = handle.get_torrent_info()
chunk_size = 500 * 1024 * 1024
my_assigned_files = []
# ⚑==========================================⚑
# HYBRID SHARDING ENGINE: Bytes vs VIP Files
# ⚑==========================================⚑
# 1. Gather all valid video files first
valid_files = []
total_valid_bytes = 0
for i in range(info.num_files()):
f_size = info.files().file_size(i)
f_name = info.files().file_path(i).lower()
if f_size > 100 * 1024 * 1024 and f_name.endswith(('.mkv', '.mp4', '.avi')) and "sample" not in f_name:
valid_files.append(i)
total_valid_bytes += f_size
# ⚑ THE SINGLE EPISODE OVERRIDE ⚑
if is_series and len(valid_files) == 1:
print("[ENGINE] πŸ“Ί Single Episode Torrent detected! Switching to 3-Engine Byte-Slicing...", flush=True)
is_series = False # Trick the engine into treating this episode like a Movie!
if is_series:
# 2. Calculate the "Perfect Balance" target size per engine
target_bytes_per_node = total_valid_bytes / total if total > 0 else total_valid_bytes
# 3. The "Greedy Bucket" Algorithm
node_assignments = {p: [] for p in range(1, total + 1)}
node_byte_totals = {p: 0 for p in range(1, total + 1)}
current_node = 1
for i in valid_files:
f_size = info.files().file_size(i)
if current_node < total and (node_byte_totals[current_node] + f_size > target_bytes_per_node) and node_byte_totals[current_node] > 0:
current_node += 1
node_assignments[current_node].append(i)
node_byte_totals[current_node] += f_size
my_assigned_files = node_assignments[part]
file_priorities = [0] * info.num_files()
for i in my_assigned_files: file_priorities[i] = 1
if priority_idx != -1 and priority_idx in my_assigned_files:
file_priorities[priority_idx] = 7
print(f"πŸš€ [NITRO BOOST] Injecting Priority 7 for File {priority_idx}!", flush=True)
handle.prioritize_files(file_priorities)
target_file_idx = priority_idx if priority_idx != -1 else (my_assigned_files[0] if my_assigned_files else 0)
print(f"πŸ“¦ [BYTE-BALANCED SHARD] Node {part}/{total} assigned {len(my_assigned_files)} files totaling {node_byte_totals[part] / 1024**3:.2f} GB.", flush=True)
else:
target_file_idx = max(range(info.num_files()), key=lambda i: info.files().file_size(i))
v_size = info.files().file_size(target_file_idx)
piece_length = info.piece_length()
total_chunks = (v_size // chunk_size) + (1 if v_size % chunk_size > 0 else 0)
chunks_per_node = total_chunks // total if total > 0 else total_chunks
start_chunk = (part - 1) * chunks_per_node
end_chunk = total_chunks if part == total else part * chunks_per_node
start_piece = (start_chunk * chunk_size) // piece_length
end_piece = ((end_chunk * chunk_size) // piece_length) + 1
if end_piece > info.num_pieces(): end_piece = info.num_pieces()
for i in range(info.num_pieces()): handle.piece_priority(i, 0)
for i in range(start_piece, end_piece): handle.piece_priority(i, 1)
print(f"🎬 [MOVIE/SINGLE SHARD] Node {part}/{total} assigned exact Chunks {start_chunk} to {end_chunk-1}", flush=True)
current_task_info[u_id]["current_file"] = os.path.basename(info.files().file_path(target_file_idx))
current_task_info[u_id]["size_gb"] = round(info.files().file_size(target_file_idx) / 1073741824, 2)
print(f"[ENGINE] Downloading: {handle.status().name}", flush=True)
last_done = 0
hard_stall_since = time.time()
soft_stall_since = None
# ⚑ THE GLOBAL STATE TRACKER (WITH LOCK) ⚑
processed_files = set()
batch_state = {'ops': [], 'paths': [], 'bytes': 0, 'last_fire': time.time(), 'queued_repo_paths': set()}
while True:
if u_id not in active_vaults: raise Exception("Task Terminated")
s = handle.status()
# ⚑==========================================⚑
# THE EVENT-DRIVEN FLOW-THROUGH UPLOADER (MULTI-SEASON UPGRADED)
# ⚑==========================================⚑
if is_series:
file_prog = handle.file_progress()
for i in my_assigned_files:
f_size = info.files().file_size(i)
if i not in processed_files and file_prog[i] == f_size:
file_path = os.path.join(work_dir, info.files().file_path(i))
if os.path.exists(file_path):
filename = os.path.basename(file_path)
# ⚑ MULTI-SEASON SCANNER: Looks for BOTH Season and Episode
match_se = re.search(r'[Ss](\d+)[^0-9]*[Ee](\d+)', filename)
if match_se:
s_num = int(match_se.group(1))
ep_num = int(match_se.group(2))
# Updates BOTH the S and the E in the Hugging Face ID
target_u_id = re.sub(r'_S\d+_E\d+_', f'_S{s_num:02d}_E{ep_num:02d}_', u_id)
print(f"⚑ [FLOW-THROUGH] S{s_num:02d}E{ep_num:02d} hit 100%. Pushing to Global Bucket...", flush=True)
else:
# Fallback for weird file names
match_e = re.search(r'[^A-Za-z0-9](\d{1,2})[^A-Za-z0-9]', filename)
ep_num = int(match_e.group(1)) if match_e else i
target_u_id = re.sub(r'_E\d+_', f'_E{ep_num:02d}_', u_id) if match_e else f"{u_id}_F{i}"
print(f"⚑ [FLOW-THROUGH] Episode {ep_num} hit 100%. Pushing to Global Bucket...", flush=True)
end_c = (f_size - 1) // chunk_size if f_size > 0 else 0
# Push into the Global Bucket
process_and_queue(file_path, target_u_id, 0, end_c, chunk_size, False, part, batch_state)
# Instantly clear raw MKV to preserve disk
try: os.remove(file_path)
except: pass
processed_files.add(i)
# ⚑ END OF DOWNLOAD FLUSH: Force commit remaining chunks only when torrent is 100% complete
if batch_state['ops'] and s.state in [lt.torrent_status.seeding, lt.torrent_status.finished]:
flush_batch(batch_state, f"Vaulting final batch for {u_id[:10]}")
if s.state in [lt.torrent_status.seeding, lt.torrent_status.finished]: break
if s.total_done > last_done:
last_done, hard_stall_since = s.total_done, time.time()
scout_status[u_id] = "active"
elif time.time() - hard_stall_since > 600:
scout_status[u_id] = "stalled"
speed = s.download_rate / 1000000
if speed < 0.1:
if soft_stall_since is None: soft_stall_since = time.time()
elif time.time() - soft_stall_since > 60:
handle.force_reannounce()
threading.Thread(target=force_inject_peers, args=(info_hash, all_trackers, handle, my_peer_id)).start()
soft_stall_since = time.time()
else: soft_stall_since = None
done_gb = s.total_done / 1073741824
total_gb = s.total_wanted / 1073741824
avail = getattr(s, 'distributed_copies', 0.0)
payload_spd = getattr(s, 'download_payload_rate', s.download_rate) / 1000000
if s.download_rate > 0:
h, rem = divmod(int((s.total_wanted - s.total_done) / s.download_rate), 3600)
m, sec = divmod(rem, 60)
eta_str = f"{h:02d}h {m:02d}m {sec:02d}s"
else:
eta_str = "Infinite"
print(f"ID: {u_id[:8]} | {s.progress*100:.1f}% ({done_gb:.2f}/{total_gb:.2f} GB) | Avail: {avail:.2f} | Seeds: {s.num_seeds} | Peers: {s.num_peers} | Speed: {speed:.2f} MB/s | Payload: {payload_spd:.2f} MB/s | ETA: {eta_str}", flush=True)
time.sleep(10)
if u_id not in active_vaults: raise Exception("Task Terminated")
# ⚑ Process Movie Shard
if not is_series:
file_path = os.path.join(work_dir, info.files().file_path(target_file_idx))
if os.path.exists(file_path):
print(f"[ENGINE] πŸš€ Target Shard reached 100%. Pushing exactly Byte Range to Global Bucket...", flush=True)
process_and_queue(file_path, u_id, start_chunk, end_chunk - 1, chunk_size, total > 1, part, batch_state)
# Final safety flush for any remaining straggler chunks
flush_batch(batch_state, f"Final flush for {u_id[:10]}")
completed_vaults.add(u_id) # ⚑ TELLS ADDON IT FINISHED EARLY
print(f"[ENGINE] βœ… Task Complete: {u_id}", flush=True)
except Exception as e: print(f"[CRITICAL ERROR/ABORT] {e}", flush=True)
finally:
if handle: ses.remove_torrent(handle)
active_handles.pop(u_id, None); current_task_info.pop(u_id, None); scout_status.pop(u_id, None)
shutil.rmtree(work_dir, ignore_errors=True)
active_vaults.discard(u_id)
@app.route('/archive')
def archive():
u_id = request.args.get('id')
info_hash = request.args.get('hash')
part = int(request.args.get('part', 1))
total = int(request.args.get('total', 1))
is_series = request.args.get('is_series') == "true"
priority_idx = int(request.args.get('priority_idx', -1))
if u_id in active_vaults: return "Already downloading", 200
active_vaults.add(u_id)
threading.Thread(target=do_archive, args=(u_id, info_hash, part, total, is_series, priority_idx)).start()
return "Started", 200
@app.route('/internal_kill/<u_id>', methods=['POST'])
def internal_kill(u_id):
check_auth()
active_vaults.discard(u_id)
return "Node Aborted", 200
@app.route('/delete_active/<u_id>', methods=['POST'])
def delete_active(u_id):
check_auth()
active_vaults.discard(u_id)
return redirect(url_for('dashboard', key=ADMIN_KEY))
@app.route('/delete_vaulted/<u_id>', methods=['POST'])
def delete_vaulted(u_id):
check_auth()
try: api.delete_folder(path_in_repo=u_id, repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
except: pass
return redirect(url_for('dashboard', key=ADMIN_KEY))
# ⚑ ADDED: BULK DELETE ROUTE FOR ENTIRE SHOWS
@app.route('/delete_show/<imdb_id>', methods=['POST'])
def delete_show(imdb_id):
check_auth()
try:
files = api.list_repo_files(repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
ops = [CommitOperationDelete(path_in_repo=f) for f in files if f.startswith(f"{imdb_id}_")]
if ops:
api.create_commit(repo_id=REPO_ID, repo_type="dataset", operations=ops, commit_message=f"Bulk delete show {imdb_id}", token=TOKEN)
except Exception as e:
print(f"[API ERROR] Bulk delete failed: {e}", flush=True)
return redirect(url_for('dashboard', key=ADMIN_KEY))
# ⚑ ADDED: BULK DELETE ROUTE FOR SPECIFIC SEASONS
@app.route('/delete_season/<imdb_id>/<season_name>', methods=['POST'])
def delete_season(imdb_id, season_name):
check_auth()
try:
files = api.list_repo_files(repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
# We need to make sure we only match the specific season (e.g. tt0212671_S04_)
season_prefix = f"{imdb_id}_{season_name}_"
ops = [CommitOperationDelete(path_in_repo=f) for f in files if f.startswith(season_prefix)]
if ops:
api.create_commit(repo_id=REPO_ID, repo_type="dataset", operations=ops, commit_message=f"Bulk delete season {season_name} for {imdb_id}", token=TOKEN)
except Exception as e:
print(f"[API ERROR] Bulk delete season failed: {e}", flush=True)
return redirect(url_for('dashboard', key=ADMIN_KEY))
if __name__ == "__main__":
app.run(host="0.0.0.0", port=7860)