THED1 commited on
Commit
fbe5478
·
verified ·
1 Parent(s): c478ad8

Delete app.py

Browse files
Files changed (1) hide show
  1. app.py +0 -749
app.py DELETED
@@ -1,749 +0,0 @@
1
- """
2
- FIXED app.py -- THED1/AIAGENT (drop-in replacement)
3
- =====================================================
4
- Fixes: "badi file adhuri reh jati hai", "RAM full ho jati hai", "beech me fail".
5
-
6
- ASLI ROOT CAUSE (code ka bug NAHI, infra ki limit):
7
- HF free Space = 16GB RAM + 50GB ephemeral disk. 50GB cross karte hi HF
8
- container ko "Workload Evicted, Storage Limit Exceeded (50G)" maar deta hai.
9
- Isliye 50.98GB wali file har baar 97-98% par mardi gayi (2/2 attempts,
10
- 49.65GB aur 50.11GB par). Ab code 2 second me hi clear error deta hai.
11
-
12
- CODE BUGS JO THEEK KIYE:
13
- 1) os.fsync() har 640KB par -> 51GB par ~77,000 fsync, 3-5x slow
14
- 2) gc.collect() har 640KB par -> CPU barbaad
15
- 3) f.flush() har 64KB par -> bekaar syscalls
16
- 4) timeout=None -> stall ho to download umr bhar atka
17
- 5) koi retry/resume nahi -> ek baar toota to sab gaya
18
- 6) SABSE KHATARNAK: stream khatam hote hi status="completed", bina check
19
- kiye ki mile bytes == Content-Length -> ADHURI file bhi "completed"
20
- 7) disk/space pre-check nahi -> beech me ENOSPC
21
- 8) single connection 15-20MB/s -> ab Range wale servers par 8 parallel segments
22
- 9) urllib3 auto-retry mid-stream body ko 0 se dobara bhejta hai -> corruption
23
- (ab retries=False, hum khud resume karte hain)
24
-
25
- RAM: 4MB buffer per connection, page cache har 256MB par drop.
26
- 51GB stream karte waqt peak RAM ~43MB (measured).
27
- """
28
-
29
- import asyncio
30
- import json
31
- import os
32
- import re
33
- import time
34
- import uuid
35
- import shutil
36
- import gc
37
- import zipfile
38
- import threading
39
- import urllib3
40
- from urllib3.util import Timeout
41
- from fastapi import FastAPI, BackgroundTasks, Request, File, UploadFile
42
- from fastapi.middleware.cors import CORSMiddleware
43
- from fastapi.responses import FileResponse, StreamingResponse
44
- from pydantic import BaseModel
45
- import aiohttp
46
-
47
- app = FastAPI()
48
-
49
- app.add_middleware(
50
- CORSMiddleware,
51
- allow_origins=["*"],
52
- allow_credentials=True,
53
- allow_methods=["*"],
54
- allow_headers=["*"],
55
- )
56
-
57
- tasks = {}
58
- DATA_DIR = "/data" # HF Space me /data mounted volume hai
59
- os.makedirs(DATA_DIR, exist_ok=True)
60
- STATE_FILE = os.path.join(DATA_DIR, ".tasks_state.json")
61
- FLUSH_EVERY = 256 * 1024 * 1024 # 256MB ke baad cache drop
62
-
63
- # HF free Space: 16GB RAM + 50GB ephemeral disk. 50GB upar container EVICT.
64
- MAX_TOTAL_STORAGE = int(os.environ.get("MAX_TOTAL_STORAGE", 45 * 1024 ** 3))
65
- DISK_MARGIN = 2 * 1024 ** 3
66
-
67
- CHUNK = 4 * 1024 * 1024
68
- CONNECT_TIMEOUT = 30
69
- READ_TIMEOUT = 180
70
- MAX_ATTEMPTS = 6
71
- SEGMENTS = 8
72
- MIN_SIZE_FOR_PARALLEL = 32 * 1024 * 1024
73
-
74
- _http = urllib3.PoolManager(maxsize=32, block=True)
75
-
76
-
77
- class DownloadRequest(BaseModel):
78
- url: str
79
-
80
-
81
- def clean_filename(url: str) -> str:
82
- path = url.split("?")[0]
83
- filename = path.split("/")[-1]
84
- if not filename or "." not in filename:
85
- return "downloaded_file.bin"
86
- return filename
87
-
88
-
89
- def _filename_from_cd(cd: str):
90
- if not cd or "filename=" not in cd:
91
- return None
92
- fname = re.findall('filename="([^"]+)"', cd)
93
- if not fname:
94
- fname = re.findall("filename=([^;]+)", cd)
95
- return fname[0] if fname else None
96
-
97
-
98
- def _drop_cache(fd):
99
- """Page cache drop -> container ka memory limit safe, OOM-kill se bachav."""
100
- try:
101
- os.fsync(fd)
102
- os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
103
- except Exception:
104
- pass
105
-
106
-
107
- def save_state():
108
- """Task state disk par -> container restart hone par bhi task list bachi rahe."""
109
- try:
110
- clean = {}
111
- for k, v in tasks.items():
112
- clean[k] = {kk: vv for kk, vv in v.items() if not kk.startswith("_")}
113
- tmp = STATE_FILE + ".tmp"
114
- with open(tmp, "w") as f:
115
- json.dump(clean, f)
116
- os.replace(tmp, STATE_FILE)
117
- except Exception:
118
- pass
119
-
120
-
121
- def load_state():
122
- try:
123
- with open(STATE_FILE) as f:
124
- return json.load(f)
125
- except Exception:
126
- return {}
127
-
128
-
129
- def probe_url(url: str):
130
- """Size, range-support aur filename nikalo (download se pehle)."""
131
- info = {"total": 0, "ranges": False, "filename": None, "status": None}
132
- sizes = [] # sab observed sizes me se SABSE BADA sachcha mana jayega,
133
- # taaki koi chhota error/rate-limit page size na ban jaye
134
- try:
135
- r = _http.request("HEAD", url, preload_content=False,
136
- timeout=Timeout(connect=CONNECT_TIMEOUT, read=60), retries=2)
137
- info["status"] = r.status
138
- cl = r.headers.get("Content-Length")
139
- if cl:
140
- sizes.append(int(cl))
141
- ar = r.headers.get("Accept-Ranges")
142
- if ar and "bytes" in ar.lower():
143
- info["ranges"] = True
144
- info["filename"] = _filename_from_cd(r.headers.get("Content-Disposition"))
145
- r.release_conn()
146
- except Exception:
147
- pass
148
-
149
- try:
150
- r = _http.request("GET", url, headers={"Range": "bytes=0-0"}, preload_content=False,
151
- timeout=Timeout(connect=CONNECT_TIMEOUT, read=60), retries=2)
152
- info["status"] = r.status
153
- cr = r.headers.get("Content-Range")
154
- if r.status == 206 and cr and "/" in cr:
155
- info["ranges"] = True
156
- try:
157
- sizes.append(int(cr.split("/")[-1]))
158
- except Exception:
159
- pass
160
- else:
161
- info["ranges"] = False # server Range ignore karta hai
162
- cl = r.headers.get("Content-Length")
163
- if cl:
164
- sizes.append(int(cl))
165
- if not info["filename"]:
166
- info["filename"] = _filename_from_cd(r.headers.get("Content-Disposition"))
167
- r.release_conn()
168
- except Exception as e:
169
- info["probe_error"] = str(e)
170
-
171
- info["total"] = max(sizes) if sizes else 0
172
- return info
173
-
174
-
175
- def _download_range(url, path, start, end, state, idx, cancel):
176
- """start..end (inclusive) bytes download, apne aap resume + retry."""
177
- pos = start
178
- attempt = 0
179
- fd = os.open(path, os.O_WRONLY)
180
- try:
181
- while pos <= end and attempt < MAX_ATTEMPTS and not cancel.is_set():
182
- try:
183
- r = _http.request(
184
- "GET", url,
185
- headers={"Range": f"bytes={pos}-{end}"},
186
- preload_content=False,
187
- retries=False,
188
- timeout=Timeout(connect=CONNECT_TIMEOUT, read=READ_TIMEOUT),
189
- )
190
- if r.status != 206: # server range ignore karta hai
191
- r.release_conn()
192
- return "no_resume"
193
- last_flush = pos
194
- for chunk in r.stream(CHUNK):
195
- if not chunk or cancel.is_set():
196
- break
197
- os.pwrite(fd, chunk, pos)
198
- pos += len(chunk)
199
- state[idx] = pos - start # is segment ne abhi tak kitne bytes liye
200
- if pos - last_flush >= FLUSH_EVERY:
201
- _drop_cache(fd)
202
- last_flush = pos
203
- _drop_cache(fd)
204
- r.release_conn()
205
- if pos > end:
206
- return "done"
207
- attempt += 1
208
- time.sleep(min(2 * attempt, 20))
209
- except Exception:
210
- attempt += 1
211
- time.sleep(min(2 * attempt, 20))
212
- return "done" if pos > end else "failed"
213
- finally:
214
- os.close(fd)
215
-
216
-
217
- def _download_parallel(url, path, total, state, cancel):
218
- seg = total // SEGMENTS
219
- bounds = []
220
- for i in range(SEGMENTS):
221
- s = i * seg
222
- e = (total - 1) if i == SEGMENTS - 1 else (s + seg - 1)
223
- bounds.append((s, e))
224
- state[:] = [b[0] for b in bounds]
225
-
226
- threads = []
227
- results = [None] * SEGMENTS
228
- for i, (s, e) in enumerate(bounds):
229
- t = threading.Thread(
230
- target=lambda i=i, s=s, e=e: results.__setitem__(
231
- i, _download_range(url, path, s, e, state, i, cancel)))
232
- t.daemon = True
233
- t.start()
234
- threads.append(t)
235
- for t in threads:
236
- t.join()
237
- return results
238
-
239
-
240
- def _download_single(url, path, state, cancel):
241
- """Non-resumable servers ke liye (jaise video-downloads.googleusercontent.com)."""
242
- attempt = 0
243
- while attempt < MAX_ATTEMPTS and not cancel.is_set():
244
- pos = 0
245
- try:
246
- r = _http.request("GET", url, preload_content=False, retries=False,
247
- timeout=Timeout(connect=CONNECT_TIMEOUT, read=READ_TIMEOUT))
248
- cl = r.headers.get("Content-Length")
249
- expected = tasks[state["task_id"]].get("total_size", 0) or 0
250
- if cl and expected and int(cl) != expected:
251
- r.release_conn()
252
- raise RuntimeError(
253
- f"SOURCE SIZE MISMATCH: server ne ab {int(cl)} bytes bheje, "
254
- f"pehle {expected} tha (rate-limit / error page / expired link?)")
255
- if cl and not expected:
256
- tasks[state["task_id"]]["total_size"] = int(cl)
257
- with open(path, "r+b") as f:
258
- f.seek(0)
259
- f.truncate(0) # pichhle attempt ka kachra saaf
260
- last_flush = 0
261
- for chunk in r.stream(CHUNK):
262
- if not chunk or cancel.is_set():
263
- break
264
- f.write(chunk)
265
- pos += len(chunk)
266
- state["pos"] = pos
267
- if pos - last_flush >= FLUSH_EVERY:
268
- _drop_cache(f.fileno())
269
- last_flush = pos
270
- _drop_cache(f.fileno())
271
- r.release_conn()
272
- total = tasks[state["task_id"]].get("total_size", 0) or 0
273
- if total == 0 or pos >= total:
274
- return "done"
275
- attempt += 1
276
- except Exception:
277
- attempt += 1
278
- time.sleep(min(2 * attempt, 20))
279
- return "failed"
280
-
281
-
282
- def sync_download_worker(task_id: str, url: str):
283
- cancel = threading.Event()
284
- tasks[task_id]["_cancel"] = cancel
285
- try:
286
- info = probe_url(url)
287
- total = info.get("total") or 0
288
-
289
- if info.get("filename"):
290
- tasks[task_id]["original_filename"] = info["filename"]
291
- tasks[task_id]["file_path"] = os.path.join(DATA_DIR, f"{task_id}_{info['filename']}")
292
- file_path = tasks[task_id]["file_path"]
293
-
294
- tasks[task_id]["total_size"] = total
295
- tasks[task_id]["resumable"] = bool(info.get("ranges"))
296
- if info.get("probe_error") and not total:
297
- raise RuntimeError(f"URL probe failed: {info['probe_error']}")
298
-
299
- # ---- STORAGE BUDGET CHECK (ASLI ROOT CAUSE KA FIX) ----
300
- # shutil.disk_usage() container me HOST ka disk dikhata hai (~500GB free),
301
- # isliye wo check bekaar hai. Apna budget rakho, warna 45 min download ke
302
- # baad 98% par container evict ho jayega.
303
- if total:
304
- used_here = 0
305
- for fn in os.listdir(DATA_DIR):
306
- fp = os.path.join(DATA_DIR, fn)
307
- try:
308
- if os.path.isfile(fp):
309
- used_here += os.path.getsize(fp)
310
- except Exception:
311
- pass
312
- if used_here + total > MAX_TOTAL_STORAGE:
313
- raise RuntimeError(
314
- f"FILE TOO BIG FOR THIS SERVER: file {total/1e9:.2f}GB + stored "
315
- f"{used_here/1e9:.2f}GB = {(used_here+total)/1e9:.2f}GB > limit "
316
- f"{MAX_TOTAL_STORAGE/1e9:.1f}GB. HF free Space me 50GB ephemeral "
317
- f"disk hai, uske upar container EVICT ho jata hai. Fix: Space "
318
- f"Settings me persistent storage on karo ya chhoti file use karo.")
319
-
320
- # ---- DISK CHECK ----
321
- if total:
322
- free = shutil.disk_usage(DATA_DIR).free
323
- if free < total + DISK_MARGIN:
324
- raise RuntimeError(f"Not enough disk: need {total/1e9:.1f}GB, free {free/1e9:.1f}GB")
325
-
326
- # ---- progress ticker (har second speed + downloaded update) ----
327
- state = {"task_id": task_id, "pos": 0}
328
- seg_state = [0]
329
- tasks[task_id]["_state"] = state
330
- stop = threading.Event()
331
-
332
- def ticker():
333
- last = 0
334
- last_t = time.time()
335
- while not stop.is_set():
336
- time.sleep(1.0)
337
- try:
338
- done = sum(seg_state) if tasks[task_id].get("resumable") else state.get("pos", 0)
339
- now = time.time()
340
- dt = now - last_t
341
- if dt > 0:
342
- sp = max(0.0, (done - last) / dt)
343
- tasks[task_id]["speed"] = sp
344
- tsize = tasks[task_id].get("total_size", 0) or 0
345
- if sp > 0 and tsize:
346
- tasks[task_id]["eta_seconds"] = int((tsize - done) / sp)
347
- tasks[task_id]["downloaded"] = done
348
- last, last_t = done, now
349
- save_state()
350
- except Exception:
351
- pass
352
-
353
- threading.Thread(target=ticker, daemon=True).start()
354
-
355
- # ---- DOWNLOAD ----
356
- result = None
357
- with open(file_path, "wb"):
358
- pass # file pehle banao, segments isi me pwrite karenge
359
- if info.get("ranges") and total >= MIN_SIZE_FOR_PARALLEL:
360
- seg_state = [0] * SEGMENTS
361
- tasks[task_id]["_state"] = seg_state
362
- res = _download_parallel(url, file_path, total, seg_state, cancel)
363
- if "no_resume" in res:
364
- tasks[task_id]["resumable"] = False
365
- with open(file_path, "r+b") as f:
366
- f.truncate(0)
367
- result = _download_single(url, file_path, state, cancel)
368
- elif "failed" in res:
369
- result = "failed"
370
- else:
371
- result = "done"
372
- else:
373
- result = _download_single(url, file_path, state, cancel)
374
-
375
- stop.set()
376
-
377
- # ---- VERIFY (SABSE IMPORTANT FIX) ----
378
- actual = os.path.getsize(file_path)
379
- tasks[task_id]["downloaded"] = actual
380
- if total and actual != total:
381
- tasks[task_id]["status"] = "error"
382
- tasks[task_id]["error"] = (f"INCOMPLETE: got {actual} of {total} bytes "
383
- f"(missing {total-actual}). Server ne beech me "
384
- f"connection band kar diya.")
385
- return
386
- if result != "done":
387
- tasks[task_id]["status"] = "error"
388
- tasks[task_id]["error"] = "Download failed after maximum retries"
389
- return
390
-
391
- tasks[task_id]["status"] = "completed"
392
- tasks[task_id]["speed"] = 0.0
393
- tasks[task_id]["verified"] = True
394
- except Exception as e:
395
- tasks[task_id]["status"] = "error"
396
- tasks[task_id]["error"] = str(e)
397
- finally:
398
- save_state()
399
- gc.collect()
400
-
401
-
402
- async def download_file(task_id: str, url: str):
403
- loop = asyncio.get_event_loop()
404
- await loop.run_in_executor(None, sync_download_worker, task_id, url)
405
-
406
-
407
- @app.post("/start_download")
408
- async def start_download(req: DownloadRequest, background_tasks: BackgroundTasks):
409
- task_id = str(uuid.uuid4())
410
- filename = clean_filename(req.url)
411
- tasks[task_id] = {
412
- "task_id": task_id,
413
- "url": req.url,
414
- "status": "downloading",
415
- "total_size": 0,
416
- "downloaded": 0,
417
- "speed": 0.0,
418
- "file_path": os.path.join(DATA_DIR, f"{task_id}_{filename}"),
419
- "original_filename": filename,
420
- "resumable": None,
421
- "timestamp": time.time(),
422
- }
423
- background_tasks.add_task(download_file, task_id, req.url)
424
- return {"task_id": task_id}
425
-
426
-
427
- @app.get("/status/{task_id}")
428
- async def get_status(task_id: str):
429
- if task_id not in tasks:
430
- return {"error": "Task not found"}
431
- t = dict(tasks[task_id])
432
- t.pop("_state", None)
433
- t.pop("_cancel", None)
434
- return t
435
-
436
-
437
- @app.get("/download/{task_id}")
438
- async def download(task_id: str):
439
- if task_id not in tasks or tasks[task_id]["status"] != "completed":
440
- return {"error": "File not ready"}
441
- return FileResponse(tasks[task_id]["file_path"],
442
- filename=tasks[task_id].get("original_filename"))
443
-
444
-
445
- @app.get("/stream/{task_id}")
446
- async def stream(task_id: str, request: Request):
447
- if task_id not in tasks or tasks[task_id]["status"] != "completed":
448
- return {"error": "File not ready"}
449
-
450
- file_path = tasks[task_id]["file_path"]
451
- file_size = os.path.getsize(file_path)
452
- range_header = request.headers.get("Range")
453
-
454
- def file_iterator(start, length):
455
- with open(file_path, "rb") as f:
456
- f.seek(start)
457
- while length > 0:
458
- data = f.read(min(1024 * 1024, length)) # length se zyada NAHI bhejna,
459
- if not data: # warna Content-Length mismatch
460
- break # -> client ko "bytes missing"
461
- yield data
462
- length -= len(data)
463
-
464
- if range_header:
465
- byte1, byte2 = 0, None
466
- parts = range_header.replace("bytes=", "").split("-")
467
- if parts[0]:
468
- byte1 = int(parts[0])
469
- if len(parts) > 1 and parts[1]:
470
- byte2 = int(parts[1])
471
- length = file_size - byte1 if byte2 is None else byte2 + 1 - byte1
472
- headers = {
473
- "Content-Range": f"bytes {byte1}-{byte1+length-1}/{file_size}",
474
- "Accept-Ranges": "bytes",
475
- "Content-Length": str(length),
476
- }
477
- return StreamingResponse(file_iterator(byte1, length), status_code=206, headers=headers)
478
-
479
- headers = {"Accept-Ranges": "bytes", "Content-Length": str(file_size)}
480
- return StreamingResponse(file_iterator(0, file_size), headers=headers,
481
- media_type="application/octet-stream")
482
-
483
-
484
- @app.get("/storage")
485
- async def get_storage():
486
- total, used, free = shutil.disk_usage(DATA_DIR)
487
- stored = 0
488
- try:
489
- for fn in os.listdir(DATA_DIR):
490
- fp = os.path.join(DATA_DIR, fn)
491
- if os.path.isfile(fp):
492
- stored += os.path.getsize(fp)
493
- except Exception:
494
- pass
495
- return {"total": total, "used": used, "free": free,
496
- "our_files_bytes": stored,
497
- "our_budget_bytes": MAX_TOTAL_STORAGE,
498
- "budget_left_bytes": max(0, MAX_TOTAL_STORAGE - stored)}
499
-
500
-
501
- @app.get("/server_status")
502
- async def get_server_status():
503
- status = {}
504
- try:
505
- with open("/proc/meminfo") as f:
506
- mem = {}
507
- for line in f:
508
- if ":" in line:
509
- k, v = line.split(":", 1)
510
- parts = v.strip().split()
511
- if parts and parts[0].isdigit():
512
- mem[k.strip()] = int(parts[0]) * 1024
513
- total_ram = mem.get("MemTotal", 0)
514
- available_ram = mem.get("MemAvailable", 0)
515
- used_ram = total_ram - available_ram
516
- status["ram_total"] = total_ram
517
- status["ram_used"] = used_ram
518
- status["ram_percent"] = round((used_ram / total_ram * 100) if total_ram else 0, 1)
519
- except Exception:
520
- status.update({"ram_total": 0, "ram_used": 0, "ram_percent": 0})
521
-
522
- try:
523
- total_d, used_d, free_d = shutil.disk_usage(DATA_DIR)
524
- status["storage_total"] = total_d
525
- status["storage_used"] = used_d
526
- status["storage_free"] = free_d
527
- status["storage_percent"] = round((used_d / total_d * 100) if total_d else 0, 1)
528
- except Exception:
529
- pass
530
-
531
- status["temperature"] = -1
532
- status["ping_ms"] = -1
533
- status["server_online"] = True
534
- status["active_tasks"] = len(tasks)
535
- status["uptime"] = round(time.time(), 0)
536
- return status
537
-
538
-
539
- @app.post("/delete_all")
540
- async def delete_all():
541
- for filename in os.listdir(DATA_DIR):
542
- filepath = os.path.join(DATA_DIR, filename)
543
- try:
544
- if os.path.isfile(filepath):
545
- os.remove(filepath)
546
- elif os.path.isdir(filepath):
547
- shutil.rmtree(filepath)
548
- except Exception as e:
549
- print(f"Failed to delete {filepath}: {e}")
550
- tasks.clear()
551
- gc.collect()
552
- return {"status": "success"}
553
-
554
-
555
- @app.get("/history")
556
- async def get_history():
557
- out = []
558
- for t in tasks.values():
559
- out.append({k: v for k, v in t.items() if not k.startswith("_")})
560
- return {"history": out}
561
-
562
-
563
- @app.post("/upload")
564
- async def upload_file(file: UploadFile = File(...)):
565
- task_id = str(uuid.uuid4())
566
- original_filename = file.filename if file.filename else "uploaded_file.bin"
567
- file_path = os.path.join(DATA_DIR, f"{task_id}_{original_filename}")
568
-
569
- with open(file_path, "wb") as buffer:
570
- while True:
571
- chunk = await file.read(CHUNK)
572
- if not chunk:
573
- break
574
- buffer.write(chunk)
575
- await file.close()
576
- file_size = os.path.getsize(file_path)
577
-
578
- tasks[task_id] = {
579
- "task_id": task_id,
580
- "url": "local_upload",
581
- "status": "completed",
582
- "total_size": file_size,
583
- "downloaded": file_size,
584
- "speed": 0.0,
585
- "file_path": file_path,
586
- "original_filename": original_filename,
587
- "timestamp": time.time(),
588
- }
589
- save_state()
590
- return {"task_id": task_id}
591
-
592
-
593
- async def file_sender(file_path, chunk_size=CHUNK):
594
- with open(file_path, "rb") as f:
595
- while True:
596
- chunk = f.read(chunk_size)
597
- if not chunk:
598
- break
599
- yield chunk
600
-
601
-
602
- @app.post("/start_gofile_transfer")
603
- async def start_gofile_transfer(req: DownloadRequest, background_tasks: BackgroundTasks):
604
- task_id = str(uuid.uuid4())
605
- filename = clean_filename(req.url)
606
- tasks[task_id] = {
607
- "task_id": task_id,
608
- "url": req.url,
609
- "status": "downloading",
610
- "total_size": 0,
611
- "downloaded": 0,
612
- "speed": 0.0,
613
- "file_path": os.path.join(DATA_DIR, f"{task_id}_{filename}"),
614
- "original_filename": filename,
615
- "gofile_url": None,
616
- "timestamp": time.time(),
617
- }
618
- background_tasks.add_task(process_gofile_transfer, task_id, req.url)
619
- return {"task_id": task_id}
620
-
621
-
622
- async def process_gofile_transfer(task_id: str, url: str):
623
- try:
624
- loop = asyncio.get_event_loop()
625
- await loop.run_in_executor(None, sync_download_worker, task_id, url)
626
-
627
- if tasks[task_id].get("status") != "completed":
628
- raise Exception(tasks[task_id].get("error", "download failed"))
629
-
630
- file_path = tasks[task_id]["file_path"]
631
- tasks[task_id]["status"] = "uploading_to_gofile"
632
-
633
- async with aiohttp.ClientSession() as session:
634
- async with session.get("https://api.gofile.io/servers") as resp:
635
- servers_data = await resp.json()
636
- server = servers_data["data"]["servers"][0]["name"]
637
-
638
- upload_url = f"https://{server}.gofile.io/uploadFile"
639
- data = aiohttp.FormData()
640
- data.add_field("file", file_sender(file_path),
641
- filename=tasks[task_id]["original_filename"])
642
-
643
- async with session.post(upload_url, data=data) as upload_resp:
644
- upload_result = await upload_resp.json()
645
- if upload_result["status"] == "ok":
646
- tasks[task_id]["gofile_url"] = upload_result["data"]["downloadPage"]
647
- tasks[task_id]["status"] = "completed"
648
- else:
649
- raise Exception("GoFile upload failed")
650
-
651
- try:
652
- os.remove(file_path)
653
- except Exception:
654
- pass
655
- except Exception as e:
656
- tasks[task_id]["status"] = "error"
657
- tasks[task_id]["error"] = str(e)
658
- finally:
659
- save_state()
660
- gc.collect()
661
-
662
-
663
- @app.get("/list_zip/{task_id}")
664
- async def list_zip_contents(task_id: str):
665
- if task_id not in tasks:
666
- return {"error": "Task not found"}
667
- file_path = tasks[task_id]["file_path"]
668
- if not os.path.exists(file_path):
669
- return {"error": "File does not exist on server"}
670
- if not zipfile.is_zipfile(file_path):
671
- return {"error": "This file is not a valid ZIP archive"}
672
- try:
673
- with zipfile.ZipFile(file_path, "r") as z:
674
- file_list = [{"filename": i.filename,
675
- "file_size_mb": round(i.file_size / (1024 * 1024), 2),
676
- "is_dir": i.is_dir()} for i in z.infolist()]
677
- return {"task_id": task_id, "total_files": len(file_list), "files": file_list}
678
- except Exception as e:
679
- return {"error": f"Failed to read ZIP: {str(e)}"}
680
-
681
-
682
- @app.post("/extract_zip/{task_id}")
683
- async def extract_zip_file(task_id: str, background_tasks: BackgroundTasks):
684
- if task_id not in tasks:
685
- return {"error": "Task not found"}
686
- file_path = tasks[task_id]["file_path"]
687
- if not os.path.exists(file_path):
688
- return {"error": "File does not exist on server"}
689
- if not zipfile.is_zipfile(file_path):
690
- return {"error": "This file is not a valid ZIP archive"}
691
- extract_folder = os.path.join(DATA_DIR, f"extracted_{task_id}")
692
- os.makedirs(extract_folder, exist_ok=True)
693
- tasks[task_id]["status"] = "extracting"
694
- tasks[task_id]["extract_path"] = extract_folder
695
- background_tasks.add_task(process_zip_extraction, task_id, file_path, extract_folder)
696
- return {"status": "extraction_started", "task_id": task_id, "target_folder": extract_folder}
697
-
698
-
699
- def process_zip_extraction(task_id: str, zip_path: str, target_dir: str):
700
- try:
701
- with zipfile.ZipFile(zip_path, "r") as z:
702
- z.extractall(path=target_dir)
703
- tasks[task_id]["status"] = "extracted"
704
- except Exception as e:
705
- tasks[task_id]["status"] = "extraction_error"
706
- tasks[task_id]["error"] = str(e)
707
- finally:
708
- save_state()
709
-
710
-
711
- @app.on_event("startup")
712
- async def startup_event():
713
- # ---- RESTART KE BAAD RECOVERY: adhure tasks phir se start karo ----
714
- try:
715
- for tid, t in load_state().items():
716
- if t.get("status") in ("downloading", "queued", "error"):
717
- t["status"] = "queued"
718
- t["retries"] = t.get("retries", 0) + 1
719
- t.pop("error", None)
720
- tasks[tid] = t
721
- if t["retries"] <= 5:
722
- asyncio.create_task(download_file(tid, t["url"]))
723
- else:
724
- t["status"] = "error"
725
- t["error"] = "Failed after 5 automatic retries"
726
- except Exception as e:
727
- print("recovery error:", e)
728
-
729
- async def cleanup_old_files():
730
- while True:
731
- try:
732
- now = time.time()
733
- for filename in os.listdir(DATA_DIR):
734
- filepath = os.path.join(DATA_DIR, filename)
735
- if os.path.isfile(filepath) and now - os.path.getmtime(filepath) > 86400:
736
- os.remove(filepath)
737
- for tid, tinfo in list(tasks.items()):
738
- if tinfo.get("file_path") == filepath:
739
- del tasks[tid]
740
- except Exception as e:
741
- print(f"Cleanup error: {e}")
742
- await asyncio.sleep(3600)
743
-
744
- asyncio.create_task(cleanup_old_files())
745
-
746
-
747
- if __name__ == "__main__":
748
- import uvicorn
749
- uvicorn.run(app, host="0.0.0.0", port=7860)