amri12310 commited on
Commit
520f009
·
verified ·
1 Parent(s): b096de5

Update engine.py

Browse files
Files changed (1) hide show
  1. engine.py +84 -51
engine.py CHANGED
@@ -1,98 +1,141 @@
1
- import os, threading, shutil, json, time, glob, subprocess, urllib.parse
2
  import warnings, requests, struct, socket
3
  import libtorrent as lt
4
- from flask import Flask, request, Response
5
  from huggingface_hub import HfApi
6
 
7
- # --- CONFIGURATION ---
8
- REPO_ID = "amri12310/hf-vault-99281-xk9-p02-mlq8-r4x-v9928-zb-v2-alpha-sync"
9
- TOKEN = os.environ.get("HF_TOKEN")
 
10
 
11
  app = Flask(__name__)
12
  api = HfApi()
 
 
 
 
 
13
  active_vaults = set()
14
- active_handles = {} # Registry for targeted peer injection
 
 
 
 
 
 
 
 
15
 
16
- # --- OPTIMIZED ENGINE SETTINGS ---
 
 
17
  settings = {
18
  'listen_interfaces': '0.0.0.0:6881',
19
  'enable_dht': True,
 
20
  'connections_limit': 350,
 
 
21
  'aio_threads': 16,
22
- 'max_queued_disk_bytes': 512 * 1024 * 1024, # 512MB Buffer
 
23
  'peer_connect_timeout': 8,
24
  'announce_to_all_trackers': True,
25
- 'enable_outgoing_utp': False, # Force TCP to bypass HF firewall
26
  'enable_incoming_utp': False,
27
  }
28
  ses = lt.session(settings)
29
 
 
 
 
30
  def trigger_github_scout(u_id, info_hash):
31
  """Automatically deploys the GitHub Scout via API"""
32
- gh_token = os.environ.get("GH_PAT")
33
-
34
- # ⚠️ UPDATE THESE THREE VARIABLES ⚠️
35
- gh_owner = "amrit123101"
36
- gh_repo = "Scout"
37
- workflow_filename = "peer_radar.yml"
38
-
39
- if not gh_token:
40
  print("⚠️ [API] GitHub Token missing. Scout deployment skipped.", flush=True)
41
  return
42
 
43
- url = f"https://api.github.com/repos/{gh_owner}/{gh_repo}/actions/workflows/{workflow_filename}/dispatches"
44
  headers = {
45
  "Accept": "application/vnd.github+json",
46
- "Authorization": f"Bearer {gh_token}",
47
  "X-GitHub-Api-Version": "2022-11-28"
48
  }
49
  payload = {
50
- "ref": "main", # Change to 'master' if your repo defaults to master
51
  "inputs": {
52
  "info_hash": info_hash,
53
  "u_id": u_id,
54
  "hf_url": f"https://{os.environ.get('SPACE_HOST', 'amri12310-sys-b-engine.hf.space')}"
55
  }
56
  }
57
-
58
  try:
59
  r = requests.post(url, headers=headers, json=payload, timeout=5)
60
  if r.status_code == 204:
61
  print(f"🚀 [COMMAND] Automated GitHub Scout deployed for {u_id}", flush=True)
62
- else:
63
- print(f"❌ [COMMAND] Scout deployment failed: {r.status_code} - {r.text}", flush=True)
64
- except Exception as e:
65
- print(f"❌ [COMMAND] Network error: {e}", flush=True)
66
 
67
  @app.route('/inject_peers', methods=['POST'])
68
  def inject_peers():
69
  """Endpoint for the GitHub Scout to push discovered UDP peers"""
70
  data = request.get_json()
71
- u_id = data.get('id')
72
- peers = data.get('peers', [])
73
-
74
  handle = active_handles.get(u_id)
75
- if not handle:
76
- return f"No active handle for {u_id}", 404
77
-
78
  count = 0
79
  for p in peers:
80
  try:
81
  handle.connect_peer((p['ip'], int(p['port'])), 0)
82
  count += 1
83
  except: continue
84
-
85
- if count > 0:
86
- print(f"🎯 [SCOUT] Force-fed {count} hidden peers into {u_id}", flush=True)
87
- return f"Injected {count} peers", 200
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
89
  def do_archive(u_id, info_hash):
90
  handle = None
91
  try:
92
  work_dir, upload_dir = f"/app/temp/{u_id}", f"/app/upload/{u_id}"
93
  os.makedirs(work_dir, exist_ok=True); os.makedirs(upload_dir, exist_ok=True)
94
 
95
- # Build Magnet
96
  r_trackers = requests.get("https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt", timeout=5)
97
  all_trackers = [t for t in r_trackers.text.split('\n') if t.startswith('http')]
98
  magnet = f"magnet:?xt=urn:btih:{info_hash}" + "".join([f"&tr={urllib.parse.quote(t)}" for t in all_trackers])
@@ -102,7 +145,7 @@ def do_archive(u_id, info_hash):
102
  handle = ses.add_torrent(atp)
103
  active_handles[u_id] = handle
104
 
105
- # Launch Scout Automatically
106
  threading.Thread(target=trigger_github_scout, args=(u_id, info_hash)).start()
107
 
108
  print(f"[ENGINE] Waiting for metadata for {u_id}...", flush=True)
@@ -111,26 +154,18 @@ def do_archive(u_id, info_hash):
111
  while True:
112
  s = handle.status()
113
  if s.state in [lt.torrent_status.seeding, lt.torrent_status.finished]: break
114
-
115
  done_gb = s.total_done / 1073741824
116
  total_gb = s.total_wanted / 1073741824
117
  speed = s.download_rate / 1000000
118
  print(f"ID: {u_id[:8]} | {s.progress*100:.1f}% ({done_gb:.2f}/{total_gb:.2f} GB) | Seeds: {s.num_seeds} | Peers: {s.num_peers} | Speed: {speed:.2f} MB/s", flush=True)
119
  time.sleep(10)
120
 
121
- # Process & Split
122
- files = []
123
- for root, _, filenames in os.walk(work_dir):
124
- for f in filenames:
125
- if f.endswith(('.mkv', '.mp4', '.avi', '.iso', '.zim', '.gguf')):
126
- files.append(os.path.join(root, f))
127
-
128
- if not files: raise Exception("No media found.")
129
  video = max(files, key=os.path.getsize)
130
  temp_path = os.path.join(upload_dir, "proc.tmp")
131
  shutil.move(video, temp_path)
132
 
133
- print(f"[ENGINE] Splitting file into 500MB chunks...", flush=True)
134
  subprocess.run(['split', '-b', '500M', '-d', '--suffix-length=2', temp_path, f"{upload_dir}/chunk_"])
135
  os.remove(temp_path)
136
 
@@ -140,11 +175,10 @@ def do_archive(u_id, info_hash):
140
  with open(f"{upload_dir}/meta.json", "w") as f:
141
  json.dump({"size": os.path.getsize(video), "chunks": len(raw_chunks), "chunk_size_mb": 500}, f)
142
 
143
- # Sync to Vault
144
  api.upload_folder(folder_path=upload_dir, path_in_repo=u_id, repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
145
- print(f"[ENGINE] ✅ Complete: {u_id}", flush=True)
146
 
147
- except Exception as e: print(f"[ERROR] {e}", flush=True)
148
  finally:
149
  if handle: ses.remove_torrent(handle)
150
  active_handles.pop(u_id, None)
@@ -154,7 +188,6 @@ def do_archive(u_id, info_hash):
154
  @app.route('/archive')
155
  def archive():
156
  u_id, info_hash = request.args.get('id'), request.args.get('hash')
157
- if not u_id or not info_hash: return "Missing params", 400
158
  if u_id in active_vaults: return "Busy", 200
159
  active_vaults.add(u_id)
160
  threading.Thread(target=do_archive, args=(u_id, info_hash)).start()
 
1
+ import os, threading, shutil, json, time, glob, subprocess, urllib.parse, logging
2
  import warnings, requests, struct, socket
3
  import libtorrent as lt
4
+ from flask import Flask, request, Response, stream_with_context
5
  from huggingface_hub import HfApi
6
 
7
+ # --- SILENCE WARNINGS ---
8
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
9
+ log = logging.getLogger('werkzeug')
10
+ log.setLevel(logging.ERROR)
11
 
12
  app = Flask(__name__)
13
  api = HfApi()
14
+
15
+ REPO_ID = "amri12310/hf-vault-99281-xk9-p02-mlq8-r4x-v9928-zb-v2-alpha-sync"
16
+ TOKEN = os.environ.get("HF_TOKEN")
17
+ GH_TOKEN = os.environ.get("GH_PAT")
18
+
19
  active_vaults = set()
20
+ active_handles = {} # Registry for peer injection
21
+
22
+ @app.after_request
23
+ def after_request(response):
24
+ response.headers.add('Access-Control-Allow-Origin', '*')
25
+ response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization,Range')
26
+ response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS')
27
+ response.headers.add('Accept-Ranges', 'bytes')
28
+ return response
29
 
30
+ # ==========================================
31
+ # THE GOLD STANDARD: I/O UNLEASHED ENGINE
32
+ # ==========================================
33
  settings = {
34
  'listen_interfaces': '0.0.0.0:6881',
35
  'enable_dht': True,
36
+ 'dht_bootstrap_nodes': 'router.bittorrent.com:6881,router.utorrent.com:6881,dht.transmissionbt.com:6881',
37
  'connections_limit': 350,
38
+ 'unchoke_slots_limit': 150,
39
+ 'active_downloads': 1,
40
  'aio_threads': 16,
41
+ 'cache_size': 65536,
42
+ 'max_queued_disk_bytes': 512 * 1024 * 1024,
43
  'peer_connect_timeout': 8,
44
  'announce_to_all_trackers': True,
45
+ 'enable_outgoing_utp': False, # Force TCP for Muscle
46
  'enable_incoming_utp': False,
47
  }
48
  ses = lt.session(settings)
49
 
50
+ # ==========================================
51
+ # GITHUB SCOUT AUTOMATION
52
+ # ==========================================
53
  def trigger_github_scout(u_id, info_hash):
54
  """Automatically deploys the GitHub Scout via API"""
55
+ if not GH_TOKEN:
 
 
 
 
 
 
 
56
  print("⚠️ [API] GitHub Token missing. Scout deployment skipped.", flush=True)
57
  return
58
 
59
+ url = f"https://api.github.com/repos/amrit123101/Scout/actions/workflows/peer_radar.yml/dispatches"
60
  headers = {
61
  "Accept": "application/vnd.github+json",
62
+ "Authorization": f"Bearer {GH_TOKEN}",
63
  "X-GitHub-Api-Version": "2022-11-28"
64
  }
65
  payload = {
66
+ "ref": "main",
67
  "inputs": {
68
  "info_hash": info_hash,
69
  "u_id": u_id,
70
  "hf_url": f"https://{os.environ.get('SPACE_HOST', 'amri12310-sys-b-engine.hf.space')}"
71
  }
72
  }
 
73
  try:
74
  r = requests.post(url, headers=headers, json=payload, timeout=5)
75
  if r.status_code == 204:
76
  print(f"🚀 [COMMAND] Automated GitHub Scout deployed for {u_id}", flush=True)
77
+ except: pass
 
 
 
78
 
79
  @app.route('/inject_peers', methods=['POST'])
80
  def inject_peers():
81
  """Endpoint for the GitHub Scout to push discovered UDP peers"""
82
  data = request.get_json()
83
+ u_id, peers = data.get('id'), data.get('peers', [])
 
 
84
  handle = active_handles.get(u_id)
85
+ if not handle: return "No handle", 404
 
 
86
  count = 0
87
  for p in peers:
88
  try:
89
  handle.connect_peer((p['ip'], int(p['port'])), 0)
90
  count += 1
91
  except: continue
92
+ if count > 0: print(f"🎯 [SCOUT] Injected {count} peers into {u_id}", flush=True)
93
+ return "OK", 200
94
+
95
+ # ==========================================
96
+ # STREAMING FRONTEND (STREMIO COMPATIBLE)
97
+ # ==========================================
98
+ @app.route('/stream/<u_id>')
99
+ def stream_vault(u_id):
100
+ auth = {"Authorization": f"Bearer {TOKEN}"}
101
+ meta_url = f"https://huggingface.co/datasets/{REPO_ID}/resolve/main/{u_id}/meta.json"
102
+ try:
103
+ meta = requests.get(meta_url, headers=auth, timeout=10).json()
104
+ total_size = meta.get('size', 0)
105
+ num_chunks, c_size_mb = meta.get('chunks', 0), meta.get('chunk_size_mb', 500)
106
+ except: return "Metadata not found", 404
107
+
108
+ range_header = request.headers.get('Range', None)
109
+ byte_start = 0
110
+ if range_header:
111
+ try: byte_start = int(range_header.replace('bytes=', '').split('-')[0])
112
+ except: pass
113
 
114
+ def generate():
115
+ chunk_size_bytes = c_size_mb * 1024 * 1024
116
+ start_chunk_idx = byte_start // chunk_size_bytes
117
+ for i in range(start_chunk_idx, num_chunks):
118
+ chunk_url = f"https://huggingface.co/datasets/{REPO_ID}/resolve/main/{u_id}/{i}.bin"
119
+ headers = auth.copy()
120
+ if i == start_chunk_idx and byte_start % chunk_size_bytes > 0:
121
+ headers["Range"] = f"bytes={byte_start % chunk_size_bytes}-"
122
+ with requests.get(chunk_url, headers=headers, stream=True) as r:
123
+ for data in r.iter_content(chunk_size=1024*512): yield data
124
+
125
+ headers = {"Content-Length": str(total_size - byte_start), "Content-Type": "video/x-matroska", "Accept-Ranges": "bytes"}
126
+ if range_header: headers["Content-Range"] = f"bytes {byte_start}-{total_size-1}/{total_size}"
127
+ return Response(stream_with_context(generate()), status=206 if range_header else 200, headers=headers)
128
+
129
+ # ==========================================
130
+ # CORE ENGINE LOGIC
131
+ # ==========================================
132
  def do_archive(u_id, info_hash):
133
  handle = None
134
  try:
135
  work_dir, upload_dir = f"/app/temp/{u_id}", f"/app/upload/{u_id}"
136
  os.makedirs(work_dir, exist_ok=True); os.makedirs(upload_dir, exist_ok=True)
137
 
138
+ # Build Tracker Bomb
139
  r_trackers = requests.get("https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt", timeout=5)
140
  all_trackers = [t for t in r_trackers.text.split('\n') if t.startswith('http')]
141
  magnet = f"magnet:?xt=urn:btih:{info_hash}" + "".join([f"&tr={urllib.parse.quote(t)}" for t in all_trackers])
 
145
  handle = ses.add_torrent(atp)
146
  active_handles[u_id] = handle
147
 
148
+ # Launch Scout
149
  threading.Thread(target=trigger_github_scout, args=(u_id, info_hash)).start()
150
 
151
  print(f"[ENGINE] Waiting for metadata for {u_id}...", flush=True)
 
154
  while True:
155
  s = handle.status()
156
  if s.state in [lt.torrent_status.seeding, lt.torrent_status.finished]: break
 
157
  done_gb = s.total_done / 1073741824
158
  total_gb = s.total_wanted / 1073741824
159
  speed = s.download_rate / 1000000
160
  print(f"ID: {u_id[:8]} | {s.progress*100:.1f}% ({done_gb:.2f}/{total_gb:.2f} GB) | Seeds: {s.num_seeds} | Peers: {s.num_peers} | Speed: {speed:.2f} MB/s", flush=True)
161
  time.sleep(10)
162
 
163
+ # Splitting & Syncing
164
+ files = [os.path.join(r, f) for r, d, fs in os.walk(work_dir) for f in fs if f.endswith(('.mkv', '.mp4', '.avi', '.iso', '.zim', '.gguf'))]
 
 
 
 
 
 
165
  video = max(files, key=os.path.getsize)
166
  temp_path = os.path.join(upload_dir, "proc.tmp")
167
  shutil.move(video, temp_path)
168
 
 
169
  subprocess.run(['split', '-b', '500M', '-d', '--suffix-length=2', temp_path, f"{upload_dir}/chunk_"])
170
  os.remove(temp_path)
171
 
 
175
  with open(f"{upload_dir}/meta.json", "w") as f:
176
  json.dump({"size": os.path.getsize(video), "chunks": len(raw_chunks), "chunk_size_mb": 500}, f)
177
 
 
178
  api.upload_folder(folder_path=upload_dir, path_in_repo=u_id, repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
179
+ print(f"[ENGINE] ✅ Vaulting complete for {u_id}", flush=True)
180
 
181
+ except Exception as e: print(f"[CRITICAL ERROR] {e}", flush=True)
182
  finally:
183
  if handle: ses.remove_torrent(handle)
184
  active_handles.pop(u_id, None)
 
188
  @app.route('/archive')
189
  def archive():
190
  u_id, info_hash = request.args.get('id'), request.args.get('hash')
 
191
  if u_id in active_vaults: return "Busy", 200
192
  active_vaults.add(u_id)
193
  threading.Thread(target=do_archive, args=(u_id, info_hash)).start()