kovinape commited on
Commit
dc63684
Β·
verified Β·
1 Parent(s): 5b64edc

Upload app.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. app.py +142 -13
app.py CHANGED
@@ -1,4 +1,4 @@
1
- import os, uuid, re, shutil, traceback, asyncio, json, time, subprocess
2
  from fastapi import FastAPI, UploadFile, File, Request
3
  from fastapi.responses import HTMLResponse, JSONResponse
4
  from huggingface_hub import batch_bucket_files
@@ -21,7 +21,7 @@ body{font-family:system-ui;max-width:900px;margin:40px auto;padding:20px;backgro
21
  #progress{width:100%;height:24px;background:#e5e7eb;border-radius:12px;overflow:hidden;margin:12px 0;display:none}
22
  #bar{height:100%;width:0%;background:#3b82f6;transition:width 0.3s}
23
  #log{white-space:pre-wrap;background:#1e293b;color:#e2e8f0;padding:12px;border-radius:8px;max-height:300px;overflow:auto}
24
- input[type=text]{width:80%;padding:10px;border:1px solid #cbd5e1;border-radius:8px}
25
  button{background:#3b82f6;color:white;border:0;padding:10px 18px;border-radius:8px;cursor:pointer}
26
  button:hover{background:#2563eb}
27
  </style>
@@ -29,18 +29,25 @@ button:hover{background:#2563eb}
29
  <body>
30
  <h1>πŸ“€ ULP β†’ Private Bucket <code>kovinape/ulp-bucket</code></h1>
31
  <div class="card">
32
- <h3>1. Upload .txt from your PC</h3>
33
  <div class="drop" id="drop">Drop .txt here or click<br><small>100GB, 5GB shards, private bucket</small><input type="file" id="file" accept=".txt" style="display:none"></div>
34
  <div id="progress"><div id="bar"></div></div>
35
  <div id="status"></div>
36
  </div>
37
  <div class="card">
38
- <h3>2. Fetch from Link (direct upload, no VPS)</h3>
39
- <input type="text" id="url" placeholder="https://cdn2.linkforge.dpdns.org/download/AgADLg143789 or https://.../file.txt" style="width:75%">
40
  <button onclick="fetchLink()">Fetch β†’ Bucket</button>
41
  <div id="fetchStatus" style="margin-top:10px"></div>
42
  <div id="fetchBar" style="width:100%;height:24px;background:#e5e7eb;border-radius:12px;overflow:hidden;margin:10px 0;display:none"><div id="fetchBarInner" style="height:100%;width:0%;background:#10b981;transition:width 0.3s"></div></div>
43
  </div>
 
 
 
 
 
 
 
44
  <pre id="log"></pre>
45
  <script>
46
  const drop=document.getElementById('drop'), fileInput=document.getElementById('file'), bar=document.getElementById('bar'), progress=document.getElementById('progress'), status=document.getElementById('status'), log=document.getElementById('log');
@@ -67,18 +74,27 @@ async function handle(file){
67
  async function fetchLink(){
68
  const url=document.getElementById('url').value.trim();
69
  if(!url){alert('Paste a URL');return;}
70
- document.getElementById('fetchStatus').textContent='πŸ”— Fetching β†’ bucket (live progress below)...';
71
  document.getElementById('fetchBar').style.display='block';
72
  const res=await fetch('/fetch', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({url})});
73
  const data=await res.json();
74
- if(data.job_id){ poll(data.job_id); } else { document.getElementById('fetchStatus').textContent=JSON.stringify(data); }
75
  }
76
- async function poll(job_id){
77
- const bar=document.getElementById('fetchBarInner'), st=document.getElementById('fetchStatus');
 
 
 
 
 
 
 
 
 
78
  const iv=setInterval(async()=>{
79
  const r=await fetch('/fetch/progress?job_id='+job_id);
80
  const j=await r.json();
81
- st.textContent=j.status+ (j.pct?` ${j.pct}%`:'') + (j.speed?` β€’ ${j.speed}`:'');
82
  if(j.pct) bar.style.width=j.pct+'%';
83
  if(j.status.includes('Done')||j.status.includes('Failed')) clearInterval(iv);
84
  },1000);
@@ -144,7 +160,6 @@ def do_fetch(job_id, url):
144
  try:
145
  tmp_base=f"/tmp/fetch_{job_id}"
146
  os.makedirs(tmp_base, exist_ok=True)
147
- # handle mediafire page -> extract direct link
148
  orig_url=url
149
  if "mediafire.com/file/" in url:
150
  try:
@@ -159,7 +174,6 @@ def do_fetch(job_id, url):
159
  fname=url.split("/")[-1].split("?")[0] or orig_url.split("/")[-1].split("?")[0] or "fetched.txt"
160
  fname=re.sub(r'[^A-Za-z0-9._ -]', '_', fname)[:80] or "fetched.txt"
161
  if not fname.endswith(".txt"): fname+=".txt"
162
- # try to get better filename from content-disposition via HEAD
163
  try:
164
  import requests
165
  h=requests.head(url, allow_redirects=True, timeout=10)
@@ -170,7 +184,6 @@ def do_fetch(job_id, url):
170
  except: pass
171
  tmp_path=os.path.join(tmp_base, fname)
172
  jobs[job_id]={"status":f"Downloading {fname}...","pct":5}
173
- # Try aria2c 16x for speed, fallback to curl
174
  try:
175
  cmd=["aria2c","-x","16","-s","16","-k","1M","-o",tmp_path,url,"--allow-overwrite=true","--auto-file-renaming=false","--console-log-level=warn"]
176
  proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -226,6 +239,110 @@ def do_fetch(job_id, url):
226
  traceback.print_exc()
227
  jobs[job_id]={"status":f"Failed: {e}","pct":0}
228
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
229
  @app.post("/fetch")
230
  async def fetch_url(request: Request):
231
  data=await request.json()
@@ -238,6 +355,18 @@ async def fetch_url(request: Request):
238
  threading.Thread(target=do_fetch, args=(job_id, url), daemon=True).start()
239
  return {"job_id":job_id,"status":"Started"}
240
 
 
 
 
 
 
 
 
 
 
 
 
 
241
  @app.get("/fetch/progress")
242
  async def fetch_progress(job_id: str):
243
  return jobs.get(job_id, {"status":"Unknown job","pct":0})
 
1
+ import os, uuid, re, shutil, traceback, time, subprocess, json
2
  from fastapi import FastAPI, UploadFile, File, Request
3
  from fastapi.responses import HTMLResponse, JSONResponse
4
  from huggingface_hub import batch_bucket_files
 
21
  #progress{width:100%;height:24px;background:#e5e7eb;border-radius:12px;overflow:hidden;margin:12px 0;display:none}
22
  #bar{height:100%;width:0%;background:#3b82f6;transition:width 0.3s}
23
  #log{white-space:pre-wrap;background:#1e293b;color:#e2e8f0;padding:12px;border-radius:8px;max-height:300px;overflow:auto}
24
+ input[type=text]{width:75%;padding:10px;border:1px solid #cbd5e1;border-radius:8px}
25
  button{background:#3b82f6;color:white;border:0;padding:10px 18px;border-radius:8px;cursor:pointer}
26
  button:hover{background:#2563eb}
27
  </style>
 
29
  <body>
30
  <h1>πŸ“€ ULP β†’ Private Bucket <code>kovinape/ulp-bucket</code></h1>
31
  <div class="card">
32
+ <h3>1. Upload .txt from PC</h3>
33
  <div class="drop" id="drop">Drop .txt here or click<br><small>100GB, 5GB shards, private bucket</small><input type="file" id="file" accept=".txt" style="display:none"></div>
34
  <div id="progress"><div id="bar"></div></div>
35
  <div id="status"></div>
36
  </div>
37
  <div class="card">
38
+ <h3>2. Fetch from Link (direct, no VPS)</h3>
39
+ <input type="text" id="url" placeholder="https://cdn2.linkforge.dpdns.org/download/AgADLg143789" style="width:75%">
40
  <button onclick="fetchLink()">Fetch β†’ Bucket</button>
41
  <div id="fetchStatus" style="margin-top:10px"></div>
42
  <div id="fetchBar" style="width:100%;height:24px;background:#e5e7eb;border-radius:12px;overflow:hidden;margin:10px 0;display:none"><div id="fetchBarInner" style="height:100%;width:0%;background:#10b981;transition:width 0.3s"></div></div>
43
  </div>
44
+ <div class="card">
45
+ <h3>3. Fetch from GoFile (direct to bucket)</h3>
46
+ <input type="text" id="gofile" placeholder="https://gofile.io/d/oF3BUlMO" style="width:75%">
47
+ <button onclick="fetchGofile()">Fetch GoFile β†’ Bucket</button>
48
+ <div id="gofileStatus" style="margin-top:10px"></div>
49
+ <div id="gofileBar" style="width:100%;height:24px;background:#e5e7eb;border-radius:12px;overflow:hidden;margin:10px 0;display:none"><div id="gofileBarInner" style="height:100%;width:0%;background:#ff6b35;transition:width 0.3s"></div></div>
50
+ </div>
51
  <pre id="log"></pre>
52
  <script>
53
  const drop=document.getElementById('drop'), fileInput=document.getElementById('file'), bar=document.getElementById('bar'), progress=document.getElementById('progress'), status=document.getElementById('status'), log=document.getElementById('log');
 
74
  async function fetchLink(){
75
  const url=document.getElementById('url').value.trim();
76
  if(!url){alert('Paste a URL');return;}
77
+ document.getElementById('fetchStatus').textContent='πŸ”— Fetching β†’ bucket...';
78
  document.getElementById('fetchBar').style.display='block';
79
  const res=await fetch('/fetch', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({url})});
80
  const data=await res.json();
81
+ if(data.job_id){ poll(data.job_id,'fetch'); } else { document.getElementById('fetchStatus').textContent=JSON.stringify(data); }
82
  }
83
+ async function fetchGofile(){
84
+ const url=document.getElementById('gofile').value.trim();
85
+ if(!url){alert('Paste GoFile URL');return;}
86
+ document.getElementById('gofileStatus').textContent='πŸ”— Resolving GoFile...';
87
+ document.getElementById('gofileBar').style.display='block';
88
+ const res=await fetch('/fetch/gofile', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({url})});
89
+ const data=await res.json();
90
+ if(data.job_id){ poll(data.job_id,'gofile'); } else { document.getElementById('gofileStatus').textContent=JSON.stringify(data); }
91
+ }
92
+ async function poll(job_id,type='fetch'){
93
+ const bar=document.getElementById(type=='gofile'?'gofileBarInner':'fetchBarInner'), st=document.getElementById(type=='gofile'?'gofileStatus':'fetchStatus');
94
  const iv=setInterval(async()=>{
95
  const r=await fetch('/fetch/progress?job_id='+job_id);
96
  const j=await r.json();
97
+ st.textContent=j.status+ (j.pct?` ${j.pct}%`:'');
98
  if(j.pct) bar.style.width=j.pct+'%';
99
  if(j.status.includes('Done')||j.status.includes('Failed')) clearInterval(iv);
100
  },1000);
 
160
  try:
161
  tmp_base=f"/tmp/fetch_{job_id}"
162
  os.makedirs(tmp_base, exist_ok=True)
 
163
  orig_url=url
164
  if "mediafire.com/file/" in url:
165
  try:
 
174
  fname=url.split("/")[-1].split("?")[0] or orig_url.split("/")[-1].split("?")[0] or "fetched.txt"
175
  fname=re.sub(r'[^A-Za-z0-9._ -]', '_', fname)[:80] or "fetched.txt"
176
  if not fname.endswith(".txt"): fname+=".txt"
 
177
  try:
178
  import requests
179
  h=requests.head(url, allow_redirects=True, timeout=10)
 
184
  except: pass
185
  tmp_path=os.path.join(tmp_base, fname)
186
  jobs[job_id]={"status":f"Downloading {fname}...","pct":5}
 
187
  try:
188
  cmd=["aria2c","-x","16","-s","16","-k","1M","-o",tmp_path,url,"--allow-overwrite=true","--auto-file-renaming=false","--console-log-level=warn"]
189
  proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
239
  traceback.print_exc()
240
  jobs[job_id]={"status":f"Failed: {e}","pct":0}
241
 
242
+ def do_gofile(job_id, url):
243
+ import hashlib, json as js
244
+ jobs[job_id]={"status":"Resolving GoFile...","pct":2}
245
+ try:
246
+ m=re.search(r"gofile\.io/d/([A-Za-z0-9-]+)", url)
247
+ if not m:
248
+ jobs[job_id]={"status":"Invalid GoFile URL","pct":0}; return
249
+ content_id=m.group(1)
250
+ GOFILE_UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
251
+ GOFILE_LANG="en-US"
252
+ GOFILE_SALT="12af056dacea0b"
253
+ def website_token(acc, offset=0):
254
+ import hashlib as hl, time as tm
255
+ window=int(tm.time()//14400)+offset
256
+ raw=f"{GOFILE_UA}::{GOFILE_LANG}::{acc}::{window}::{GOFILE_SALT}"
257
+ return hl.sha256(raw.encode()).hexdigest()
258
+ # guest token
259
+ out=subprocess.run(["curl","-sS","-X","POST","https://api.gofile.io/accounts","-H",f"User-Agent: {GOFILE_UA}","-H","Origin: https://gofile.io","-d",""], capture_output=True, text=True, timeout=30)
260
+ token=(js.loads(out.stdout).get("data",{}) or {}).get("token")
261
+ if not token:
262
+ jobs[job_id]={"status":"No guest token","pct":0}; return
263
+ jobs[job_id]={"status":"Guest token ok, resolving...","pct":10}
264
+ def contents(acc, off, cid):
265
+ cmd=["curl","-sS","-G",f"https://api.gofile.io/contents/{cid}","-H",f"Authorization: Bearer {acc}","-H",f"X-Website-Token: {website_token(acc,off)}","-H",f"X-BL: {GOFILE_LANG}","-H",f"User-Agent: {GOFILE_UA}","-H","Accept: */*","-H","Origin: https://gofile.io","-H","Referer: https://gofile.io/","--data-urlencode","contentFilter=","--data-urlencode","page=1","--data-urlencode","pageSize=1000","--data-urlencode","sortField=createTime","--data-urlencode","sortDirection=-1"]
266
+ out=subprocess.run(cmd, capture_output=True, text=True, timeout=30)
267
+ return js.loads(out.stdout)
268
+ def collect(acc, off, cid):
269
+ out=contents(acc, off, cid)
270
+ st=out.get("status")
271
+ if st=="error-notPremium" and off==0:
272
+ out=contents(acc,-1,cid); st=out.get("status")
273
+ if st!="ok": return None, f"gofile: {st}"
274
+ d=out.get("data",{}) or {}
275
+ if d.get("type")=="file" or d.get("link"):
276
+ return [(d.get("name") or "gofile_file.txt", int(d.get("size") or 0), d.get("link") or "")], None
277
+ entries=[]
278
+ for child_id,c in (d.get("children") or {}).items():
279
+ if c.get("type")=="folder":
280
+ sub,err=collect(acc,off,child_id)
281
+ if err: return None, err
282
+ entries.extend(sub)
283
+ elif c.get("type")=="file":
284
+ entries.append((c.get("name") or "gofile_file.txt", int(c.get("size") or 0), c.get("link") or ""))
285
+ return entries, None
286
+ files,err=None,None
287
+ for off in (0,-1):
288
+ files,err=collect(token,off,content_id)
289
+ if files is not None and not err: break
290
+ if not files:
291
+ jobs[job_id]={"status":f"Failed: {err or 'no files'}","pct":0}; return
292
+ files=[(n,s,u) for n,s,u in files if u]
293
+ total=len(files)
294
+ jobs[job_id]={"status":f"Found {total} file(s), downloading...","pct":15}
295
+ for idx,(fname_raw,fsize_b,dl_url) in enumerate(files,1):
296
+ fname=re.sub(r'[^A-Za-z0-9._ -]', '_', fname_raw)[:80] or f"gofile_{idx}.txt"
297
+ tmp_base=f"/tmp/gofile_{job_id}_{idx}"
298
+ os.makedirs(tmp_base, exist_ok=True)
299
+ tmp_path=os.path.join(tmp_base, fname)
300
+ cmd=["curl","-sS","-L","-o",tmp_path,dl_url,"-H",f"Cookie: accountToken={token}","-H",f"User-Agent: {GOFILE_UA}","-H","Referer: https://gofile.io/"]
301
+ proc=subprocess.Popen(cmd)
302
+ while proc.poll() is None:
303
+ try:
304
+ cur=os.path.getsize(tmp_path) if os.path.exists(tmp_path) else 0
305
+ pct=15+ int((idx-1)/total*70 + (cur/max(1,fsize_b))*70/total) if fsize_b else 15+int((idx-1)/total*70)
306
+ jobs[job_id]={"status":f"Downloading {idx}/{total} {fname} {cur//1024//1024}MB/{fsize_b//1024//1024}MB","pct":pct}
307
+ except: pass
308
+ time.sleep(1)
309
+ if proc.returncode!=0 or not os.path.exists(tmp_path) or os.path.getsize(tmp_path)==0:
310
+ jobs[job_id]={"status":f"Failed {fname}","pct":0}; continue
311
+ size=os.path.getsize(tmp_path)
312
+ if size <= SHARD_SIZE:
313
+ batch_bucket_files(BUCKET_ID, add=[(tmp_path, fname)])
314
+ else:
315
+ with open(tmp_path,"rb") as fh:
316
+ part=0
317
+ while True:
318
+ chunk_path=f"{tmp_path}.part{part:03d}"
319
+ with open(chunk_path,"wb") as out:
320
+ written=0
321
+ while written < SHARD_SIZE:
322
+ data=fh.read(4*1024*1024)
323
+ if not data: break
324
+ out.write(data)
325
+ written+=len(data)
326
+ if written==0:
327
+ try: os.remove(chunk_path)
328
+ except: pass
329
+ break
330
+ if not os.path.exists(chunk_path) or os.path.getsize(chunk_path)==0: break
331
+ hf_name=f"{os.path.splitext(fname)[0]}_part{part:03d}.txt"
332
+ batch_bucket_files(BUCKET_ID, add=[(chunk_path, hf_name)])
333
+ try: os.remove(chunk_path)
334
+ except: pass
335
+ part+=1
336
+ if fh.tell()>=size: break
337
+ try: os.remove(tmp_path)
338
+ except: pass
339
+ try: shutil.rmtree(tmp_base)
340
+ except: pass
341
+ jobs[job_id]={"status":f"Done β€” {total} file(s) β†’ bucket","pct":100}
342
+ except Exception as e:
343
+ traceback.print_exc()
344
+ jobs[job_id]={"status":f"Failed: {e}","pct":0}
345
+
346
  @app.post("/fetch")
347
  async def fetch_url(request: Request):
348
  data=await request.json()
 
355
  threading.Thread(target=do_fetch, args=(job_id, url), daemon=True).start()
356
  return {"job_id":job_id,"status":"Started"}
357
 
358
+ @app.post("/fetch/gofile")
359
+ async def fetch_gofile(request: Request):
360
+ data=await request.json()
361
+ url=data.get("url","").strip()
362
+ if not url or "gofile.io" not in url:
363
+ return JSONResponse({"status":"error","error":"Invalid GoFile URL"}, status_code=400)
364
+ job_id=uuid.uuid4().hex[:8]
365
+ jobs[job_id]={"status":"Queued GoFile...","pct":0}
366
+ import threading
367
+ threading.Thread(target=do_gofile, args=(job_id, url), daemon=True).start()
368
+ return {"job_id":job_id,"status":"Started"}
369
+
370
  @app.get("/fetch/progress")
371
  async def fetch_progress(job_id: str):
372
  return jobs.get(job_id, {"status":"Unknown job","pct":0})