THED1 commited on
Commit
7cacf91
·
verified ·
1 Parent(s): fbe5478

Upload app.py

Browse files
Files changed (1) hide show
  1. app.py +748 -0
app.py ADDED
@@ -0,0 +1,748 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ FIXED app.py -- THED1/AIAGENT (drop-in replacement)
3
+ =====================================================
4
+ """
5
+
6
+ import asyncio
7
+ import json
8
+ import os
9
+ import re
10
+ import time
11
+ import uuid
12
+ import shutil
13
+ import gc
14
+ import zipfile
15
+ import threading
16
+ import urllib3
17
+ from urllib3.util import Timeout
18
+ from fastapi import FastAPI, BackgroundTasks, Request, File, UploadFile
19
+ from fastapi.middleware.cors import CORSMiddleware
20
+ from fastapi.responses import FileResponse, StreamingResponse
21
+ from pydantic import BaseModel
22
+ import aiohttp
23
+
24
+ app = FastAPI()
25
+
26
+ app.add_middleware(
27
+ CORSMiddleware,
28
+ allow_origins=["*"],
29
+ allow_credentials=True,
30
+ allow_methods=["*"],
31
+ allow_headers=["*"],
32
+ )
33
+
34
+ tasks = {}
35
+ DATA_DIR = "/data" # HF Space me /data mounted volume hai
36
+ os.makedirs(DATA_DIR, exist_ok=True)
37
+ STATE_FILE = os.path.join(DATA_DIR, ".tasks_state.json")
38
+ FLUSH_EVERY = 256 * 1024 * 1024 # 256MB ke baad cache drop
39
+
40
+ # HF free Space: 16GB RAM + 50GB ephemeral disk. 50GB upar container EVICT.
41
+ MAX_TOTAL_STORAGE = int(os.environ.get("MAX_TOTAL_STORAGE", 45 * 1024 ** 3))
42
+ DISK_MARGIN = 2 * 1024 ** 3
43
+
44
+ CHUNK = 4 * 1024 * 1024
45
+ CONNECT_TIMEOUT = 30
46
+ READ_TIMEOUT = 180
47
+ MAX_ATTEMPTS = 6
48
+ SEGMENTS = 8
49
+ MIN_SIZE_FOR_PARALLEL = 32 * 1024 * 1024
50
+
51
+ _http = urllib3.PoolManager(maxsize=32, block=True)
52
+
53
+
54
+ class DownloadRequest(BaseModel):
55
+ url: str
56
+
57
+
58
+ def clean_filename(url: str) -> str:
59
+ path = url.split("?")[0]
60
+ filename = path.split("/")[-1]
61
+ if not filename or "." not in filename:
62
+ return "downloaded_file.bin"
63
+ return filename
64
+
65
+
66
+ def _filename_from_cd(cd: str):
67
+ if not cd or "filename=" not in cd:
68
+ return None
69
+ fname = re.findall('filename="([^"]+)"', cd)
70
+ if not fname:
71
+ fname = re.findall("filename=([^;]+)", cd)
72
+ return fname[0] if fname else None
73
+
74
+
75
+ def _drop_cache(fd):
76
+ """Page cache drop -> container ka memory limit safe, OOM-kill se bachav."""
77
+ try:
78
+ os.fsync(fd)
79
+ os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
80
+ except Exception:
81
+ pass
82
+
83
+
84
+ def save_state():
85
+ """Task state disk par -> container restart hone par bhi task list bachi rahe."""
86
+ try:
87
+ clean = {}
88
+ for k, v in tasks.items():
89
+ clean[k] = {kk: vv for kk, vv in v.items() if not kk.startswith("_")}
90
+ tmp = STATE_FILE + ".tmp"
91
+ with open(tmp, "w") as f:
92
+ json.dump(clean, f)
93
+ os.replace(tmp, STATE_FILE)
94
+ except Exception:
95
+ pass
96
+
97
+
98
+ def load_state():
99
+ try:
100
+ with open(STATE_FILE) as f:
101
+ return json.load(f)
102
+ except Exception:
103
+ return {}
104
+
105
+
106
+ def probe_url(url: str):
107
+ """Size, range-support aur filename nikalo (download se pehle)."""
108
+ info = {"total": 0, "ranges": False, "filename": None, "status": None}
109
+ sizes = [] # sab observed sizes me se SABSE BADA sachcha mana jayega,
110
+ # taaki koi chhota error/rate-limit page size na ban jaye
111
+ try:
112
+ r = _http.request("HEAD", url, preload_content=False,
113
+ timeout=Timeout(connect=CONNECT_TIMEOUT, read=60), retries=2)
114
+ info["status"] = r.status
115
+ cl = r.headers.get("Content-Length")
116
+ if cl:
117
+ sizes.append(int(cl))
118
+ ar = r.headers.get("Accept-Ranges")
119
+ if ar and "bytes" in ar.lower():
120
+ info["ranges"] = True
121
+ info["filename"] = _filename_from_cd(r.headers.get("Content-Disposition"))
122
+ r.release_conn()
123
+ except Exception:
124
+ pass
125
+
126
+ try:
127
+ r = _http.request("GET", url, headers={"Range": "bytes=0-0"}, preload_content=False,
128
+ timeout=Timeout(connect=CONNECT_TIMEOUT, read=60), retries=2)
129
+ info["status"] = r.status
130
+ cr = r.headers.get("Content-Range")
131
+ if r.status == 206 and cr and "/" in cr:
132
+ info["ranges"] = True
133
+ try:
134
+ sizes.append(int(cr.split("/")[-1]))
135
+ except Exception:
136
+ pass
137
+ else:
138
+ info["ranges"] = False # server Range ignore karta hai
139
+ cl = r.headers.get("Content-Length")
140
+ if cl:
141
+ sizes.append(int(cl))
142
+ if not info["filename"]:
143
+ info["filename"] = _filename_from_cd(r.headers.get("Content-Disposition"))
144
+ r.release_conn()
145
+ except Exception as e:
146
+ info["probe_error"] = str(e)
147
+
148
+ info["total"] = max(sizes) if sizes else 0
149
+ return info
150
+
151
+
152
+ def _download_range(url, path, start, end, state, idx, cancel):
153
+ """start..end (inclusive) bytes download, apne aap resume + retry."""
154
+ pos = start
155
+ attempt = 0
156
+ fd = os.open(path, os.O_WRONLY)
157
+ try:
158
+ while pos <= end and attempt < MAX_ATTEMPTS and not cancel.is_set():
159
+ try:
160
+ r = _http.request(
161
+ "GET", url,
162
+ headers={"Range": f"bytes={pos}-{end}"},
163
+ preload_content=False,
164
+ retries=False,
165
+ timeout=Timeout(connect=CONNECT_TIMEOUT, read=READ_TIMEOUT),
166
+ )
167
+ if r.status != 206: # server range ignore karta hai
168
+ r.release_conn()
169
+ return "no_resume"
170
+ last_flush = pos
171
+ for chunk in r.stream(CHUNK):
172
+ if not chunk or cancel.is_set():
173
+ break
174
+ os.pwrite(fd, chunk, pos)
175
+ pos += len(chunk)
176
+ state[idx] = pos - start # is segment ne abhi tak kitne bytes liye
177
+ if pos - last_flush >= FLUSH_EVERY:
178
+ _drop_cache(fd)
179
+ last_flush = pos
180
+ _drop_cache(fd)
181
+ r.release_conn()
182
+ if pos > end:
183
+ return "done"
184
+ attempt += 1
185
+ time.sleep(min(2 * attempt, 20))
186
+ except Exception:
187
+ attempt += 1
188
+ time.sleep(min(2 * attempt, 20))
189
+ return "done" if pos > end else "failed"
190
+ finally:
191
+ os.close(fd)
192
+
193
+
194
+ def _download_parallel(url, path, total, state, cancel):
195
+ seg = total // SEGMENTS
196
+ bounds = []
197
+ for i in range(SEGMENTS):
198
+ s = i * seg
199
+ e = (total - 1) if i == SEGMENTS - 1 else (s + seg - 1)
200
+ bounds.append((s, e))
201
+ state[:] = [b[0] for b in bounds]
202
+
203
+ threads = []
204
+ results = [None] * SEGMENTS
205
+ for i, (s, e) in enumerate(bounds):
206
+ t = threading.Thread(
207
+ target=lambda i=i, s=s, e=e: results.__setitem__(
208
+ i, _download_range(url, path, s, e, state, i, cancel)))
209
+ t.daemon = True
210
+ t.start()
211
+ threads.append(t)
212
+ for t in threads:
213
+ t.join()
214
+ return results
215
+
216
+
217
+ def _download_single(url, path, state, cancel):
218
+ """Non-resumable servers ke liye (jaise video-downloads.googleusercontent.com)."""
219
+ attempt = 0
220
+ while attempt < MAX_ATTEMPTS and not cancel.is_set():
221
+ pos = 0
222
+ try:
223
+ r = _http.request("GET", url, preload_content=False, retries=False,
224
+ timeout=Timeout(connect=CONNECT_TIMEOUT, read=READ_TIMEOUT))
225
+ cl = r.headers.get("Content-Length")
226
+ expected = tasks[state["task_id"]].get("total_size", 0) or 0
227
+ if cl and expected and int(cl) != expected:
228
+ r.release_conn()
229
+ raise RuntimeError(
230
+ f"SOURCE SIZE MISMATCH: server ne ab {int(cl)} bytes bheje, "
231
+ f"pehle {expected} tha (rate-limit / error page / expired link?)")
232
+ if cl and not expected:
233
+ tasks[state["task_id"]]["total_size"] = int(cl)
234
+ with open(path, "r+b") as f:
235
+ f.seek(0)
236
+ f.truncate(0) # pichhle attempt ka kachra saaf
237
+ last_flush = 0
238
+ for chunk in r.stream(CHUNK):
239
+ if not chunk or cancel.is_set():
240
+ break
241
+ f.write(chunk)
242
+ pos += len(chunk)
243
+ state["pos"] = pos
244
+ if pos - last_flush >= FLUSH_EVERY:
245
+ _drop_cache(f.fileno())
246
+ last_flush = pos
247
+ _drop_cache(f.fileno())
248
+ r.release_conn()
249
+ total = tasks[state["task_id"]].get("total_size", 0) or 0
250
+ if total == 0 or pos >= total:
251
+ return "done"
252
+ attempt += 1
253
+ except Exception:
254
+ attempt += 1
255
+ time.sleep(min(2 * attempt, 20))
256
+ return "failed"
257
+
258
+
259
+ def sync_download_worker(task_id: str, url: str):
260
+ cancel = threading.Event()
261
+ tasks[task_id]["_cancel"] = cancel
262
+ try:
263
+ info = probe_url(url)
264
+ total = info.get("total") or 0
265
+
266
+ if info.get("filename"):
267
+ tasks[task_id]["original_filename"] = info["filename"]
268
+ tasks[task_id]["file_path"] = os.path.join(DATA_DIR, f"{task_id}_{info['filename']}")
269
+ file_path = tasks[task_id]["file_path"]
270
+
271
+ tasks[task_id]["total_size"] = total
272
+ tasks[task_id]["resumable"] = bool(info.get("ranges"))
273
+ if info.get("probe_error") and not total:
274
+ raise RuntimeError(f"URL probe failed: {info['probe_error']}")
275
+
276
+ # ---- STORAGE BUDGET CHECK (ASLI ROOT CAUSE KA FIX) ----
277
+ # shutil.disk_usage() container me HOST ka disk dikhata hai (~500GB free),
278
+ # isliye wo check bekaar hai. Apna budget rakho, warna 45 min download ke
279
+ # baad 98% par container evict ho jayega.
280
+ if total:
281
+ used_here = 0
282
+ for fn in os.listdir(DATA_DIR):
283
+ fp = os.path.join(DATA_DIR, fn)
284
+ try:
285
+ if os.path.isfile(fp):
286
+ used_here += os.path.getsize(fp)
287
+ except Exception:
288
+ pass
289
+ if used_here + total > MAX_TOTAL_STORAGE:
290
+ raise RuntimeError(
291
+ f"FILE TOO BIG FOR THIS SERVER: file {total/1e9:.2f}GB + stored "
292
+ f"{used_here/1e9:.2f}GB = {(used_here+total)/1e9:.2f}GB > limit "
293
+ f"{MAX_TOTAL_STORAGE/1e9:.1f}GB. HF free Space me 50GB ephemeral "
294
+ f"disk hai, uske upar container EVICT ho jata hai. Fix: Space "
295
+ f"Settings me persistent storage on karo ya chhoti file use karo.")
296
+
297
+ # ---- DISK CHECK ----
298
+ if total:
299
+ free = shutil.disk_usage(DATA_DIR).free
300
+ if free < total + DISK_MARGIN:
301
+ raise RuntimeError(f"Not enough disk: need {total/1e9:.1f}GB, free {free/1e9:.1f}GB")
302
+
303
+ # ---- progress ticker (har second speed + downloaded update) ----
304
+ state = {"task_id": task_id, "pos": 0}
305
+ seg_state = [0]
306
+ tasks[task_id]["_state"] = state
307
+ stop = threading.Event()
308
+
309
+ def ticker():
310
+ last = 0
311
+ last_t = time.time()
312
+ while not stop.is_set():
313
+ time.sleep(1.0)
314
+ try:
315
+ done = sum(seg_state) if tasks[task_id].get("resumable") else state.get("pos", 0)
316
+ now = time.time()
317
+ dt = now - last_t
318
+ if dt > 0:
319
+ sp = max(0.0, (done - last) / dt)
320
+ tasks[task_id]["speed"] = sp
321
+ tsize = tasks[task_id].get("total_size", 0) or 0
322
+ if sp > 0 and tsize:
323
+ tasks[task_id]["eta_seconds"] = int((tsize - done) / sp)
324
+ tasks[task_id]["downloaded"] = done
325
+ last, last_t = done, now
326
+ save_state()
327
+ except Exception:
328
+ pass
329
+
330
+ threading.Thread(target=ticker, daemon=True).start()
331
+
332
+ # ---- DOWNLOAD ----
333
+ result = None
334
+ with open(file_path, "wb"):
335
+ pass # file pehle banao, segments isi me pwrite karenge
336
+ if info.get("ranges") and total >= MIN_SIZE_FOR_PARALLEL:
337
+ seg_state = [0] * SEGMENTS
338
+ tasks[task_id]["_state"] = seg_state
339
+ res = _download_parallel(url, file_path, total, seg_state, cancel)
340
+ if "no_resume" in res:
341
+ tasks[task_id]["resumable"] = False
342
+ with open(file_path, "r+b") as f:
343
+ f.truncate(0)
344
+ result = _download_single(url, file_path, state, cancel)
345
+ elif "failed" in res:
346
+ result = "failed"
347
+ else:
348
+ result = "done"
349
+ else:
350
+ result = _download_single(url, file_path, state, cancel)
351
+
352
+ stop.set()
353
+
354
+ # ---- VERIFY (SABSE IMPORTANT FIX) ----
355
+ actual = os.path.getsize(file_path)
356
+ tasks[task_id]["downloaded"] = actual
357
+ if total and actual != total:
358
+ tasks[task_id]["status"] = "error"
359
+ tasks[task_id]["error"] = (f"INCOMPLETE: got {actual} of {total} bytes "
360
+ f"(missing {total-actual}). Server ne beech me "
361
+ f"connection band kar diya.")
362
+ return
363
+ if result != "done":
364
+ tasks[task_id]["status"] = "error"
365
+ tasks[task_id]["error"] = "Download failed after maximum retries"
366
+ return
367
+
368
+ tasks[task_id]["status"] = "completed"
369
+ tasks[task_id]["speed"] = 0.0
370
+ tasks[task_id]["verified"] = True
371
+ except Exception as e:
372
+ tasks[task_id]["status"] = "error"
373
+ tasks[task_id]["error"] = str(e)
374
+ finally:
375
+ save_state()
376
+ gc.collect()
377
+
378
+
379
+ async def download_file(task_id: str, url: str):
380
+ loop = asyncio.get_event_loop()
381
+ await loop.run_in_executor(None, sync_download_worker, task_id, url)
382
+
383
+
384
+ @app.post("/start_download")
385
+ async def start_download(req: DownloadRequest, background_tasks: BackgroundTasks):
386
+ task_id = str(uuid.uuid4())
387
+ filename = clean_filename(req.url)
388
+ tasks[task_id] = {
389
+ "task_id": task_id,
390
+ "url": req.url,
391
+ "status": "downloading",
392
+ "total_size": 0,
393
+ "downloaded": 0,
394
+ "speed": 0.0,
395
+ "file_path": os.path.join(DATA_DIR, f"{task_id}_{filename}"),
396
+ "original_filename": filename,
397
+ "resumable": None,
398
+ "timestamp": time.time(),
399
+ }
400
+ background_tasks.add_task(download_file, task_id, req.url)
401
+ return {"task_id": task_id}
402
+
403
+
404
+ @app.get("/status/{task_id}")
405
+ async def get_status(task_id: str):
406
+ if task_id not in tasks:
407
+ return {"error": "Task not found"}
408
+ t = dict(tasks[task_id])
409
+ t.pop("_state", None)
410
+ t.pop("_cancel", None)
411
+ return t
412
+
413
+
414
+ @app.get("/download/{task_id}")
415
+ async def download(task_id: str):
416
+ if task_id not in tasks or tasks[task_id]["status"] != "completed":
417
+ return {"error": "File not ready"}
418
+ return FileResponse(tasks[task_id]["file_path"],
419
+ filename=tasks[task_id].get("original_filename"))
420
+
421
+
422
+ @app.get("/stream/{task_id}")
423
+ async def stream(task_id: str, request: Request):
424
+ if task_id not in tasks or tasks[task_id]["status"] != "completed":
425
+ return {"error": "File not ready"}
426
+
427
+ file_path = tasks[task_id]["file_path"]
428
+ file_size = os.path.getsize(file_path)
429
+ range_header = request.headers.get("Range")
430
+
431
+ def file_iterator(start, length):
432
+ with open(file_path, "rb") as f:
433
+ f.seek(start)
434
+ while length > 0:
435
+ data = f.read(min(1024 * 1024, length)) # length se zyada NAHI bhejna,
436
+ if not data: # warna Content-Length mismatch
437
+ break # -> client ko "bytes missing"
438
+ yield data
439
+ length -= len(data)
440
+
441
+ if range_header:
442
+ byte1, byte2 = 0, None
443
+ parts = range_header.replace("bytes=", "").split("-")
444
+ if parts[0]:
445
+ byte1 = int(parts[0])
446
+ if len(parts) > 1 and parts[1]:
447
+ byte2 = int(parts[1])
448
+ length = file_size - byte1 if byte2 is None else byte2 + 1 - byte1
449
+ headers = {
450
+ "Content-Range": f"bytes {byte1}-{byte1+length-1}/{file_size}",
451
+ "Accept-Ranges": "bytes",
452
+ "Content-Length": str(length),
453
+ }
454
+ return StreamingResponse(file_iterator(byte1, length), status_code=206, headers=headers)
455
+
456
+ headers = {"Accept-Ranges": "bytes", "Content-Length": str(file_size)}
457
+ return StreamingResponse(file_iterator(0, file_size), headers=headers,
458
+ media_type="application/octet-stream")
459
+
460
+
461
+ @app.get("/storage")
462
+ async def get_storage():
463
+ total, used, free = shutil.disk_usage(DATA_DIR)
464
+ stored = 0
465
+ try:
466
+ for fn in os.listdir(DATA_DIR):
467
+ fp = os.path.join(DATA_DIR, fn)
468
+ if os.path.isfile(fp):
469
+ stored += os.path.getsize(fp)
470
+ except Exception:
471
+ pass
472
+ return {"total": total, "used": used, "free": free,
473
+ "our_files_bytes": stored,
474
+ "our_budget_bytes": MAX_TOTAL_STORAGE,
475
+ "budget_left_bytes": max(0, MAX_TOTAL_STORAGE - stored)}
476
+
477
+
478
+ @app.get("/server_status")
479
+ async def get_server_status():
480
+ status = {}
481
+ try:
482
+ with open("/proc/meminfo") as f:
483
+ mem = {}
484
+ for line in f:
485
+ if ":" in line:
486
+ k, v = line.split(":", 1)
487
+ parts = v.strip().split()
488
+ if parts and parts[0].isdigit():
489
+ mem[k.strip()] = int(parts[0]) * 1024
490
+ total_ram = mem.get("MemTotal", 0)
491
+ available_ram = mem.get("MemAvailable", 0)
492
+ used_ram = total_ram - available_ram
493
+ status["ram_total"] = total_ram
494
+ status["ram_used"] = used_ram
495
+ status["ram_percent"] = round((used_ram / total_ram * 100) if total_ram else 0, 1)
496
+ except Exception:
497
+ status.update({"ram_total": 0, "ram_used": 0, "ram_percent": 0})
498
+
499
+ try:
500
+ total_d, used_d, free_d = shutil.disk_usage(DATA_DIR)
501
+ status["storage_total"] = total_d
502
+ status["storage_used"] = used_d
503
+ status["storage_free"] = free_d
504
+ status["storage_percent"] = round((used_d / total_d * 100) if total_d else 0, 1)
505
+ except Exception:
506
+ pass
507
+
508
+ status["temperature"] = -1
509
+ status["ping_ms"] = -1
510
+ status["server_online"] = True
511
+ status["active_tasks"] = len(tasks)
512
+ status["uptime"] = round(time.time(), 0)
513
+ return status
514
+
515
+
516
+ @app.post("/delete_all")
517
+ async def delete_all():
518
+ for filename in os.listdir(DATA_DIR):
519
+ filepath = os.path.join(DATA_DIR, filename)
520
+ try:
521
+ if os.path.isfile(filepath):
522
+ os.remove(filepath)
523
+ elif os.path.isdir(filepath):
524
+ shutil.rmtree(filepath)
525
+ except Exception as e:
526
+ print(f"Failed to delete {filepath}: {e}")
527
+ tasks.clear()
528
+ gc.collect()
529
+ return {"status": "success"}
530
+
531
+
532
+ @app.get("/history")
533
+ async def get_history():
534
+ out = []
535
+ for t in tasks.values():
536
+ out.append({k: v for k, v in t.items() if not k.startswith("_")})
537
+ return {"history": out}
538
+
539
+
540
+ @app.post("/upload")
541
+ async def upload_file(file: UploadFile = File(...)):
542
+ task_id = str(uuid.uuid4())
543
+ original_filename = file.filename if file.filename else "uploaded_file.bin"
544
+ file_path = os.path.join(DATA_DIR, f"{task_id}_{original_filename}")
545
+
546
+ with open(file_path, "wb") as buffer:
547
+ while True:
548
+ chunk = await file.read(CHUNK)
549
+ if not chunk:
550
+ break
551
+ buffer.write(chunk)
552
+ await file.close()
553
+ file_size = os.path.getsize(file_path)
554
+
555
+ tasks[task_id] = {
556
+ "task_id": task_id,
557
+ "url": "local_upload",
558
+ "status": "completed",
559
+ "total_size": file_size,
560
+ "downloaded": file_size,
561
+ "speed": 0.0,
562
+ "file_path": file_path,
563
+ "original_filename": original_filename,
564
+ "timestamp": time.time(),
565
+ }
566
+ save_state()
567
+ return {"task_id": task_id}
568
+
569
+
570
+ async def file_sender(file_path, chunk_size=CHUNK):
571
+ with open(file_path, "rb") as f:
572
+ while True:
573
+ chunk = f.read(chunk_size)
574
+ if not chunk:
575
+ break
576
+ yield chunk
577
+
578
+
579
+ @app.post("/start_gofile_transfer")
580
+ async def start_gofile_transfer(req: DownloadRequest, background_tasks: BackgroundTasks):
581
+ task_id = str(uuid.uuid4())
582
+ filename = clean_filename(req.url)
583
+ tasks[task_id] = {
584
+ "task_id": task_id,
585
+ "url": req.url,
586
+ "status": "downloading",
587
+ "total_size": 0,
588
+ "downloaded": 0,
589
+ "speed": 0.0,
590
+ "file_path": os.path.join(DATA_DIR, f"{task_id}_{filename}"),
591
+ "original_filename": filename,
592
+ "gofile_url": None,
593
+ "timestamp": time.time(),
594
+ }
595
+ background_tasks.add_task(process_gofile_transfer, task_id, req.url)
596
+ return {"task_id": task_id}
597
+
598
+
599
+ async def process_gofile_transfer(task_id: str, url: str):
600
+ try:
601
+ loop = asyncio.get_event_loop()
602
+ await loop.run_in_executor(None, sync_download_worker, task_id, url)
603
+
604
+ if tasks[task_id].get("status") != "completed":
605
+ raise Exception(tasks[task_id].get("error", "download failed"))
606
+
607
+ file_path = tasks[task_id]["file_path"]
608
+ tasks[task_id]["status"] = "uploading_to_gofile"
609
+
610
+ async with aiohttp.ClientSession() as session:
611
+ async with session.get("https://api.gofile.io/servers") as resp:
612
+ servers_data = await resp.json()
613
+ server = servers_data["data"]["servers"][0]["name"]
614
+
615
+ upload_url = f"https://{server}.gofile.io/uploadFile"
616
+ data = aiohttp.FormData()
617
+ data.add_field("file", file_sender(file_path),
618
+ filename=tasks[task_id]["original_filename"])
619
+
620
+ async with session.post(upload_url, data=data) as upload_resp:
621
+ upload_result = await upload_resp.json()
622
+ if upload_result["status"] == "ok":
623
+ tasks[task_id]["gofile_url"] = upload_result["data"]["downloadPage"]
624
+ tasks[task_id]["status"] = "completed"
625
+ else:
626
+ raise Exception("GoFile upload failed")
627
+
628
+ try:
629
+ os.remove(file_path)
630
+ except Exception:
631
+ pass
632
+ except Exception as e:
633
+ tasks[task_id]["status"] = "error"
634
+ tasks[task_id]["error"] = str(e)
635
+ finally:
636
+ save_state()
637
+ gc.collect()
638
+
639
+
640
+ @app.get("/list_zip/{task_id}")
641
+ async def list_zip_contents(task_id: str):
642
+ if task_id not in tasks:
643
+ return {"error": "Task not found"}
644
+ file_path = tasks[task_id]["file_path"]
645
+ if not os.path.exists(file_path):
646
+ return {"error": "File does not exist on server"}
647
+ if not zipfile.is_zipfile(file_path):
648
+ return {"error": "This file is not a valid ZIP archive"}
649
+ try:
650
+ with zipfile.ZipFile(file_path, "r") as z:
651
+ file_list = [{"filename": i.filename,
652
+ "file_size_mb": round(i.file_size / (1024 * 1024), 2),
653
+ "is_dir": i.is_dir()} for i in z.infolist()]
654
+ return {"task_id": task_id, "total_files": len(file_list), "files": file_list}
655
+ except Exception as e:
656
+ return {"error": f"Failed to read ZIP: {str(e)}"}
657
+
658
+
659
+ @app.post("/extract_zip/{task_id}")
660
+ async def extract_zip_file(task_id: str, background_tasks: BackgroundTasks):
661
+ if task_id not in tasks:
662
+ return {"error": "Task not found"}
663
+ file_path = tasks[task_id]["file_path"]
664
+ if not os.path.exists(file_path):
665
+ return {"error": "File does not exist on server"}
666
+ if not zipfile.is_zipfile(file_path):
667
+ return {"error": "This file is not a valid ZIP archive"}
668
+ extract_folder = os.path.join(DATA_DIR, f"extracted_{task_id}")
669
+ os.makedirs(extract_folder, exist_ok=True)
670
+ tasks[task_id]["status"] = "extracting"
671
+ tasks[task_id]["extract_path"] = extract_folder
672
+ background_tasks.add_task(process_zip_extraction, task_id, file_path, extract_folder)
673
+ return {"status": "extraction_started", "task_id": task_id, "target_folder": extract_folder}
674
+
675
+
676
+ def process_zip_extraction(task_id: str, zip_path: str, target_dir: str):
677
+ try:
678
+ with zipfile.ZipFile(zip_path, "r") as z:
679
+ z.extractall(path=target_dir)
680
+ tasks[task_id]["status"] = "extracted"
681
+ except Exception as e:
682
+ tasks[task_id]["status"] = "extraction_error"
683
+ tasks[task_id]["error"] = str(e)
684
+ finally:
685
+ save_state()
686
+
687
+
688
+ @app.on_event("startup")
689
+ async def startup_event():
690
+ async def cleanup_old_files():
691
+ while True:
692
+ try:
693
+ now = time.time()
694
+ for filename in os.listdir(DATA_DIR):
695
+ filepath = os.path.join(DATA_DIR, filename)
696
+ if os.path.isfile(filepath) and now - os.path.getmtime(filepath) > 86400:
697
+ os.remove(filepath)
698
+ for tid, tinfo in list(tasks.items()):
699
+ if tinfo.get("file_path") == filepath:
700
+ del tasks[tid]
701
+ except Exception as e:
702
+ print(f"Cleanup error: {e}")
703
+ await asyncio.sleep(3600)
704
+
705
+ asyncio.create_task(cleanup_old_files())
706
+
707
+
708
+ if __name__ == "__main__":
709
+ import uvicorn
710
+ uvicorn.run(app, host="0.0.0.0", port=7860)
711
+ action_started", "task_id": task_id, "target_folder": extract_folder}
712
+
713
+
714
+ def process_zip_extraction(task_id: str, zip_path: str, target_dir: str):
715
+ try:
716
+ with zipfile.ZipFile(zip_path, "r") as z:
717
+ z.extractall(path=target_dir)
718
+ tasks[task_id]["status"] = "extracted"
719
+ except Exception as e:
720
+ tasks[task_id]["status"] = "extraction_error"
721
+ tasks[task_id]["error"] = str(e)
722
+ finally:
723
+ save_state()
724
+
725
+
726
+ @app.on_event("startup")
727
+ async def startup_event():
728
+ async def cleanup_old_files():
729
+ while True:
730
+ try:
731
+ now = time.time()
732
+ for filename in os.listdir(DATA_DIR):
733
+ filepath = os.path.join(DATA_DIR, filename)
734
+ if os.path.isfile(filepath) and now - os.path.getmtime(filepath) > 86400:
735
+ os.remove(filepath)
736
+ for tid, tinfo in list(tasks.items()):
737
+ if tinfo.get("file_path") == filepath:
738
+ del tasks[tid]
739
+ except Exception as e:
740
+ print(f"Cleanup error: {e}")
741
+ await asyncio.sleep(3600)
742
+
743
+ asyncio.create_task(cleanup_old_files())
744
+
745
+
746
+ if __name__ == "__main__":
747
+ import uvicorn
748
+ uvicorn.run(app, host="0.0.0.0", port=7860)