THED2 commited on
Commit
3edbce3
·
verified ·
1 Parent(s): 7f9fcbf

Delete app.py

Browse files
Files changed (1) hide show
  1. app.py +0 -517
app.py DELETED
@@ -1,517 +0,0 @@
1
- import asyncio
2
- import os
3
- import re
4
- import time
5
- import uuid
6
- import shutil
7
- import aiohttp
8
- import gc # Memory management ke liye garbage collector
9
- import zipfile # ZIP file handle karne ke liye in-built library
10
- import urllib3 # Zero-buffer strict low RAM downloding ke liye
11
- from fastapi import FastAPI, BackgroundTasks, Request, File, UploadFile
12
- from fastapi.middleware.cors import CORSMiddleware
13
- from fastapi.responses import FileResponse, StreamingResponse
14
- from pydantic import BaseModel
15
- from concurrent.futures import ThreadPoolExecutor
16
-
17
- app = FastAPI()
18
-
19
- app.add_middleware(
20
- CORSMiddleware,
21
- allow_origins=["*"],
22
- allow_credentials=True,
23
- allow_methods=["*"],
24
- allow_headers=["*"],
25
- )
26
-
27
- tasks = {}
28
- DATA_DIR = "/data"
29
- os.makedirs(DATA_DIR, exist_ok=True)
30
-
31
- # Parallel chunks handle karne ke liye separate dedicated pool executor
32
- thread_executor = ThreadPoolExecutor(max_workers=8)
33
-
34
- class DownloadRequest(BaseModel):
35
- url: str
36
-
37
- def clean_filename(url: str) -> str:
38
- """URL se sahi extension aur filename nikalne ka jugaad"""
39
- path = url.split("?")[0]
40
- filename = path.split("/")[-1]
41
- if not filename or "." not in filename:
42
- return "downloaded_file.bin"
43
- return filename
44
-
45
- async def cleanup_old_files():
46
- while True:
47
- try:
48
- now = time.time()
49
- for filename in os.listdir(DATA_DIR):
50
- filepath = os.path.join(DATA_DIR, filename)
51
- if os.path.isfile(filepath):
52
- if now - os.path.getmtime(filepath) > 86400:
53
- os.remove(filepath)
54
- for tid, tinfo in list(tasks.items()):
55
- if tinfo.get("file_path") == filepath:
56
- del tasks[tid]
57
- except Exception as e:
58
- print(f"Cleanup error: {e}")
59
- await asyncio.sleep(3600)
60
-
61
- @app.on_event("startup")
62
- async def startup_event():
63
- asyncio.create_task(cleanup_old_files())
64
-
65
- def download_segment(url: str, start_byte: int, end_byte: int, file_path: str, task_id: str, http_pool):
66
- """Specific range ka part target file ke precise offset me write karne ka concurrent worker"""
67
- headers = {"Range": f"bytes={start_byte}-{end_byte}"}
68
- try:
69
- # Network socket parameters push karenge peak buffer utilization ke liye
70
- response = http_pool.request('GET', url, headers=headers, preload_content=False, timeout=None)
71
-
72
- # Binary file stream mode me precise positioning par write karenge
73
- with open(file_path, 'rb+') as f:
74
- f.seek(start_byte)
75
- chunk_size = 128 * 1024 # Standard network maximum transmission frame alignment
76
-
77
- for chunk in response.stream(chunk_size):
78
- if not chunk:
79
- break
80
- f.write(chunk)
81
- chunk_len = len(chunk)
82
-
83
- # Global dictionary progress data thread-safe incremental updating
84
- tasks[task_id]["downloaded"] += chunk_len
85
- tasks[task_id]["chunk_speed_counter"] += chunk_len
86
- del chunk
87
-
88
- response.release_conn()
89
- except Exception as e:
90
- tasks[task_id]["status"] = "error"
91
- tasks[task_id]["error"] = str(e)
92
-
93
- def sync_download_worker(task_id: str, url: str):
94
- """Smart dynamic downloader: Choti files single connections se aur badi files segmented parallel connections se jayengi"""
95
- file_path = tasks[task_id]["file_path"]
96
- try:
97
- # High pool allocation limit taaki multiple concurrent sockets blocking interface na karein
98
- http = urllib3.PoolManager(block=False, maxsize=10, num_pools=4)
99
-
100
- # HEAD call optimization for content metadata verification
101
- head_response = http.request('HEAD', url, timeout=10.0)
102
- total_size = int(head_response.headers.get('Content-Length', 0))
103
- tasks[task_id]["total_size"] = total_size
104
-
105
- cd = head_response.headers.get('Content-Disposition')
106
- if cd and 'filename=' in cd:
107
- fname = re.findall('filename="([^"]+)"', cd)
108
- if not fname:
109
- fname = re.findall('filename=([^;]+)', cd)
110
- if fname:
111
- new_filename = fname[0]
112
- file_path = os.path.join(DATA_DIR, f"{task_id}_{new_filename}")
113
- tasks[task_id]["file_path"] = file_path
114
- tasks[task_id]["original_filename"] = new_filename
115
-
116
- # Empty/Pre-allocated file buffer skeleton banayenge parallel offsets synchronization ke liye
117
- with open(file_path, 'wb') as f:
118
- if total_size > 0:
119
- f.truncate(total_size)
120
-
121
- # Speed measurement background metrics setup
122
- tasks[task_id]["chunk_speed_counter"] = 0
123
-
124
- # SPEED TRACER THREAD LAUNCH (Independent metrics tracking engine)
125
- def tracking_loop():
126
- last_time = time.time()
127
- while tasks[task_id]["status"] == "downloading":
128
- time.sleep(1.0)
129
- current_time = time.time()
130
- elapsed = current_time - last_time
131
- if elapsed > 0:
132
- current_speed = tasks[task_id]["chunk_speed_counter"] / elapsed
133
- tasks[task_id]["speed"] = current_speed
134
- tasks[task_id]["chunk_speed_counter"] = 0
135
- last_time = current_time
136
-
137
- import threading
138
- tracker = threading.Thread(target=tracking_loop, daemon=True)
139
- tracker.start()
140
-
141
- # SEGMENTATION LOGIC: Agar file size 20MB se badi hai aur chunk capabilities supported hain
142
- is_range_supported = head_response.headers.get('Accept-Ranges') == 'bytes' or 'Content-Length' in head_response.headers
143
-
144
- if total_size > 20 * 1024 * 1024 and is_range_supported:
145
- num_segments = 6 # Hugging Face par safest efficient thread concurrency limit
146
- segment_size = total_size // num_segments
147
- futures = []
148
-
149
- for i in range(num_segments):
150
- start = i * segment_size
151
- end = total_size - 1 if i == num_segments - 1 else (start + segment_size - 1)
152
- futures.append(thread_executor.submit(download_segment, url, start, end, file_path, task_id, http))
153
-
154
- # Sub-threads compilation resolution waiting window
155
- for fut in futures:
156
- fut.result()
157
-
158
- # Final verification sequence check
159
- if tasks[task_id]["status"] == "downloading":
160
- tasks[task_id]["status"] = "completed"
161
- else:
162
- # Fallback process standard streaming workflow agar ranges completely blank/unsupported ho
163
- response = http.request('GET', url, preload_content=False, timeout=None)
164
- downloaded = 0
165
- chunk_counter = 0
166
-
167
- with open(file_path, 'wb') as f:
168
- for chunk in response.stream(128 * 1024): # Increased to 128KB for better TCP throughput
169
- if not chunk:
170
- break
171
- f.write(chunk)
172
- len_c = len(chunk)
173
- downloaded += len_c
174
- tasks[task_id]["downloaded"] = downloaded
175
- tasks[task_id]["chunk_speed_counter"] += len_c
176
-
177
- chunk_counter += 1
178
- # Reduced excessive filesystem sync/gc overhead frequency to optimize hardware bottleneck
179
- if chunk_counter % 50 == 0:
180
- f.flush()
181
- del chunk
182
- gc.collect()
183
-
184
- response.release_conn()
185
- tasks[task_id]["status"] = "completed"
186
-
187
- tasks[task_id]["speed"] = 0.0
188
- except Exception as e:
189
- tasks[task_id]["status"] = "error"
190
- tasks[task_id]["error"] = str(e)
191
- finally:
192
- gc.collect()
193
-
194
- async def download_file(task_id: str, url: str):
195
- loop = asyncio.get_event_loop()
196
- await loop.run_in_executor(thread_executor, sync_download_worker, task_id, url)
197
-
198
- @app.post("/start_download")
199
- async def start_download(req: DownloadRequest, background_tasks: BackgroundTasks):
200
- task_id = str(uuid.uuid4())
201
- filename = clean_filename(req.url)
202
-
203
- tasks[task_id] = {
204
- "task_id": task_id,
205
- "url": req.url,
206
- "status": "downloading",
207
- "total_size": 0,
208
- "downloaded": 0,
209
- "speed": 0.0,
210
- "file_path": os.path.join(DATA_DIR, f"{task_id}_{filename}"),
211
- "original_filename": filename,
212
- "timestamp": time.time()
213
- }
214
- background_tasks.add_task(download_file, task_id, req.url)
215
- return {"task_id": task_id}
216
-
217
- @app.get("/status/{task_id}")
218
- async def get_status(task_id: str):
219
- if task_id not in tasks:
220
- return {"error": "Task not found"}
221
- return tasks[task_id]
222
-
223
- @app.get("/download/{task_id}")
224
- async def download(task_id: str):
225
- if task_id not in tasks or tasks[task_id]["status"] != "completed":
226
- return {"error": "File not ready"}
227
- return FileResponse(tasks[task_id]["file_path"], filename=tasks[task_id].get("original_filename"))
228
-
229
- @app.get("/stream/{task_id}")
230
- async def stream(task_id: str, request: Request):
231
- if task_id not in tasks or tasks[task_id]["status"] != "completed":
232
- return {"error": "File not ready"}
233
-
234
- file_path = tasks[task_id]["file_path"]
235
- file_size = os.path.getsize(file_path)
236
- range_header = request.headers.get("Range")
237
-
238
- if range_header:
239
- byte1, byte2 = 0, None
240
- match = range_header.replace("bytes=", "").split("-")
241
- if match[0]:
242
- byte1 = int(match[0])
243
- if len(match) > 1 and match[1]:
244
- byte2 = int(match[1])
245
-
246
- length = file_size - byte1
247
- if byte2 is not None:
248
- length = byte2 + 1 - byte1
249
-
250
- def file_iterator(start, length):
251
- with open(file_path, "rb") as f:
252
- f.seek(start)
253
- chunk_size = 128 * 1024
254
- while length > 0:
255
- read_size = min(chunk_size, length)
256
- data = f.read(read_size)
257
- if not data:
258
- break
259
- yield data
260
- length -= len(data)
261
- del data
262
-
263
- headers = {
264
- "Content-Range": f"bytes {byte1}-{byte1+length-1}/{file_size}",
265
- "Accept-Ranges": "bytes",
266
- "Content-Length": str(length),
267
- "X-Accel-Buffering": "no", # Proxy distribution bottleneck bypass hack
268
- "Cache-Control": "no-cache"
269
- }
270
- return StreamingResponse(file_iterator(byte1, length), status_code=206, headers=headers)
271
- else:
272
- return FileResponse(file_path, headers={"X-Accel-Buffering": "no", "Cache-Control": "no-cache"})
273
-
274
- @app.get("/storage")
275
- async def get_storage():
276
- total, used, free = shutil.disk_usage(DATA_DIR)
277
- return {"total": total, "used": used, "free": free}
278
-
279
- @app.post("/delete_all")
280
- async def delete_all():
281
- for filename in os.listdir(DATA_DIR):
282
- filepath = os.path.join(DATA_DIR, filename)
283
- try:
284
- if os.path.isfile(filepath):
285
- os.remove(filepath)
286
- elif os.path.isdir(filepath):
287
- shutil.rmtree(filepath)
288
- except Exception as e:
289
- print(f"Failed to delete {filepath}: {e}")
290
- tasks.clear()
291
- gc.collect()
292
- return {"status": "success"}
293
-
294
- @app.get("/history")
295
- async def get_history():
296
- return {"history": list(tasks.values())}
297
-
298
- @app.post("/upload")
299
- async def upload_file(file: UploadFile = File(...)):
300
- task_id = str(uuid.uuid4())
301
- original_filename = file.filename if file.filename else "uploaded_file.bin"
302
- file_path = os.path.join(DATA_DIR, f"{task_id}_{original_filename}")
303
-
304
- with open(file_path, "wb") as buffer:
305
- while True:
306
- chunk = await file.read(128 * 1024)
307
- if not chunk:
308
- break
309
- buffer.write(chunk)
310
-
311
- await file.close()
312
- file_size = os.path.getsize(file_path)
313
-
314
- tasks[task_id] = {
315
- "task_id": task_id,
316
- "url": "local_upload",
317
- "status": "completed",
318
- "total_size": file_size,
319
- "downloaded": file_size,
320
- "speed": 0.0,
321
- "file_path": file_path,
322
- "original_filename": original_filename,
323
- "timestamp": time.time()
324
- }
325
- gc.collect()
326
- return {"task_id": task_id}
327
-
328
- @app.post("/start_gofile_transfer")
329
- async def start_gofile_transfer(req: DownloadRequest, background_tasks: BackgroundTasks):
330
- task_id = str(uuid.uuid4())
331
- filename = clean_filename(req.url)
332
-
333
- tasks[task_id] = {
334
- "task_id": task_id,
335
- "url": req.url,
336
- "status": "downloading",
337
- "total_size": 0,
338
- "downloaded": 0,
339
- "speed": 0.0,
340
- "file_path": os.path.join(DATA_DIR, f"{task_id}_{filename}"),
341
- "original_filename": filename,
342
- "gofile_url": None,
343
- "timestamp": time.time()
344
- }
345
- background_tasks.add_task(process_gofile_transfer, task_id, req.url)
346
- return {"task_id": task_id}
347
-
348
- async def file_sender(file_path, chunk_size=128 * 1024):
349
- with open(file_path, 'rb') as f:
350
- while True:
351
- chunk = f.read(chunk_size)
352
- if not chunk:
353
- break
354
- yield chunk
355
- del chunk
356
-
357
- def sync_gofile_download_part(task_id: str, url: str):
358
- """Same optimized multi-segment execution injection inside secondary functions"""
359
- file_path = tasks[task_id]["file_path"]
360
- http = urllib3.PoolManager(block=False, maxsize=10, num_pools=4)
361
-
362
- head_response = http.request('HEAD', url, timeout=10.0)
363
- total_size = int(head_response.headers.get('Content-Length', 0))
364
- tasks[task_id]["total_size"] = total_size
365
-
366
- cd = head_response.headers.get('Content-Disposition')
367
- if cd and 'filename=' in cd:
368
- fname = re.findall('filename="([^"]+)"', cd)
369
- if not fname:
370
- fname = re.findall('filename=([^;]+)', cd)
371
- if fname:
372
- new_filename = fname[0]
373
- file_path = os.path.join(DATA_DIR, f"{task_id}_{new_filename}")
374
- tasks[task_id]["file_path"] = file_path
375
- tasks[task_id]["original_filename"] = new_filename
376
-
377
- with open(file_path, 'wb') as f:
378
- if total_size > 0:
379
- f.truncate(total_size)
380
-
381
- tasks[task_id]["chunk_speed_counter"] = 0
382
- def tracking_loop():
383
- last_time = time.time()
384
- while tasks[task_id]["status"] == "downloading":
385
- time.sleep(1.0)
386
- current_time = time.time()
387
- elapsed = current_time - last_time
388
- if elapsed > 0:
389
- tasks[task_id]["speed"] = tasks[task_id]["chunk_speed_counter"] / elapsed
390
- tasks[task_id]["chunk_speed_counter"] = 0
391
- last_time = current_time
392
-
393
- import threading
394
- tracker = threading.Thread(target=tracking_loop, daemon=True)
395
- tracker.start()
396
-
397
- is_range_supported = head_response.headers.get('Accept-Ranges') == 'bytes' or 'Content-Length' in head_response.headers
398
- if total_size > 20 * 1024 * 1024 and is_range_supported:
399
- num_segments = 6
400
- segment_size = total_size // num_segments
401
- futures = []
402
- for i in range(num_segments):
403
- start = i * segment_size
404
- end = total_size - 1 if i == num_segments - 1 else (start + segment_size - 1)
405
- futures.append(thread_executor.submit(download_segment, url, start, end, file_path, task_id, http))
406
- for fut in futures:
407
- fut.result()
408
- else:
409
- response = http.request('GET', url, preload_content=False, timeout=None)
410
- downloaded = 0
411
- with open(file_path, 'wb') as f:
412
- for chunk in response.stream(128 * 1024):
413
- if not chunk:
414
- break
415
- f.write(chunk)
416
- len_c = len(chunk)
417
- downloaded += len_c
418
- tasks[task_id]["downloaded"] = downloaded
419
- tasks[task_id]["chunk_speed_counter"] += len_c
420
- response.release_conn()
421
-
422
- async def process_gofile_transfer(task_id: str, url: str):
423
- try:
424
- loop = asyncio.get_event_loop()
425
- await loop.run_in_executor(thread_executor, sync_gofile_download_part, task_id, url)
426
-
427
- file_path = tasks[task_id]["file_path"]
428
- tasks[task_id]["status"] = "uploading_to_gofile"
429
- tasks[task_id]["speed"] = 0.0
430
-
431
- async with aiohttp.ClientSession() as session:
432
- async with session.get("https://api.gofile.io/servers") as resp:
433
- servers_data = await resp.json()
434
- server = servers_data["data"]["servers"][0]["name"]
435
-
436
- upload_url = f"https://{server}.gofile.io/uploadFile"
437
-
438
- data = aiohttp.FormData()
439
- data.add_field('file', file_sender(file_path), filename=tasks[task_id]["original_filename"])
440
-
441
- async with session.post(upload_url, data=data) as upload_resp:
442
- upload_result = await upload_resp.json()
443
- if upload_result["status"] == "ok":
444
- tasks[task_id]["gofile_url"] = upload_result["data"]["downloadPage"]
445
- tasks[task_id]["status"] = "completed"
446
- else:
447
- raise Exception("GoFile upload failed")
448
-
449
- try:
450
- os.remove(file_path)
451
- except:
452
- pass
453
-
454
- except Exception as e:
455
- tasks[task_id]["status"] = "error"
456
- tasks[task_id]["error"] = str(e)
457
- finally:
458
- gc.collect()
459
-
460
- # ==================== ZIP MANAGEMENT FEATURES ====================
461
-
462
- @app.get("/list_zip/{task_id}")
463
- async def list_zip_contents(task_id: str):
464
- if task_id not in tasks:
465
- return {"error": "Task not found"}
466
-
467
- file_path = tasks[task_id]["file_path"]
468
- if not os.path.exists(file_path):
469
- return {"error": "File does not exist on server"}
470
-
471
- if not zipfile.is_zipfile(file_path):
472
- return {"error": "This file is not a valid ZIP archive"}
473
-
474
- try:
475
- with zipfile.ZipFile(file_path, 'r') as z:
476
- file_list = []
477
- for info in z.infolist():
478
- file_list.append({
479
- "filename": info.filename,
480
- "file_size_mb": round(info.file_size / (1024 * 1024), 2),
481
- "is_dir": info.is_dir()
482
- })
483
- return {"task_id": task_id, "total_files": len(file_list), "files": file_list}
484
- except Exception as e:
485
- return {"error": f"Failed to read ZIP: {str(e)}"}
486
-
487
- @app.post("/extract_zip/{task_id}")
488
- async def extract_zip_file(task_id: str, background_tasks: BackgroundTasks):
489
- if task_id not in tasks:
490
- return {"error": "Task not found"}
491
-
492
- file_path = tasks[task_id]["file_path"]
493
- if not os.path.exists(file_path):
494
- return {"error": "File does not exist on server"}
495
-
496
- if not zipfile.is_zipfile(file_path):
497
- return {"error": "This file is not a valid ZIP archive"}
498
-
499
- extract_folder = os.path.join(DATA_DIR, f"extracted_{task_id}")
500
- os.makedirs(extract_folder, exist_ok=True)
501
-
502
- tasks[task_id]["status"] = "extracting"
503
- tasks[task_id]["extract_path"] = extract_folder
504
-
505
- background_tasks.add_task(process_zip_extraction, task_id, file_path, extract_folder)
506
- return {"status": "extraction_started", "task_id": task_id, "target_folder": extract_folder}
507
-
508
- def process_zip_extraction(task_id: str, zip_path: str, target_dir: str):
509
- try:
510
- with zipfile.ZipFile(zip_path, 'r') as z:
511
- file_counter = 0
512
- for member in z.namelist():
513
- z.extract(member, path=target_dir)
514
- file_counter += 1
515
- if file_counter % 50 == 0:
516
- gc.collect()
517
- tasks[task