ulp-uploader / app.py
kovinape's picture
Upload app.py with huggingface_hub
c2bcbcf verified
Raw History Blame Contribute Delete
53.2 kB
import os, uuid, re, shutil, traceback, time, subprocess, json
from urllib.parse import unquote
from fastapi import FastAPI, UploadFile, File, Request
from fastapi.responses import HTMLResponse, JSONResponse
from huggingface_hub import batch_bucket_files, hffs
from concurrent.futures import ThreadPoolExecutor, as_completed
BUCKET_ID = os.environ.get("HF_BUCKET_ID", "kovinape/ulp-bucket")
SHARD_SIZE = 5 * 1024 * 1024 * 1024
app = FastAPI()
jobs = {}
HTML = """
<!DOCTYPE html>
<html>
<head><title>ULP Hub — Upload & Extract</title>
<style>
*{box-sizing:border-box;margin:0;padding:0}
body{font-family:'Segoe UI',system-ui,sans-serif;min-height:100vh;color:#e5e9f0;background:radial-gradient(1200px 800px at 15% -10%,rgba(56,189,248,.14) 0%,transparent 60%),radial-gradient(1000px 700px at 110% 10%,rgba(56,189,248,.18) 0%,transparent 55%),linear-gradient(160deg,#0b0f17 0%,#141a26 50%,#0d1220 100%);background-attachment:fixed}
body::before{content:'';position:fixed;inset:0;pointer-events:none;z-index:-1;background:linear-gradient(180deg,rgba(255,255,255,.04),transparent 30%),repeating-linear-gradient(0deg,transparent 0 2px,rgba(255,255,255,.014) 2px 3px),linear-gradient(110deg,transparent 42%,rgba(255,255,255,.06) 50%,transparent 58%)}
h2,h3{color:#cbd5e1;font-weight:600;letter-spacing:.02em}
code{background:rgba(255,255,255,.08);padding:2px 7px;border-radius:6px;font-size:.85em;color:#7dd3fc}
nav{display:flex;gap:12px;padding:18px 22px;position:sticky;top:0;z-index:10;background:linear-gradient(180deg,rgba(15,20,30,.78),rgba(15,20,30,.55));backdrop-filter:blur(20px);-webkit-backdrop-filter:blur(20px);border-bottom:1px solid rgba(226,232,240,.12);box-shadow:0 6px 22px rgba(0,0,0,.45)}
nav button{padding:11px 20px;border:1px solid rgba(226,232,240,.16);background:linear-gradient(165deg,rgba(203,213,225,.16),rgba(100,116,139,.05));color:#aab4c6;border-radius:10px;cursor:pointer;font-size:.92rem;backdrop-filter:blur(8px);transition:.25s;box-shadow:0 2px 10px rgba(0,0,0,.4),inset 0 1px 0 rgba(255,255,255,.2),inset 0 -2px 4px rgba(0,0,0,.25)}
nav button:hover{color:#e2e8f0;border-color:rgba(125,211,252,.4);transform:translateY(-1px)}
nav button.active{background:linear-gradient(135deg,rgba(56,189,248,.3),rgba(37,99,235,.22));color:#fff;border-color:rgba(125,211,252,.6);box-shadow:0 0 18px rgba(56,189,248,.28),inset 0 1px 0 rgba(255,255,255,.25)}
.page{display:none;padding:24px;max-width:1000px;margin:0 auto}
.page.active{display:block;animation:fadein .35s}
@keyframes fadein{from{opacity:0;transform:translateY(6px)}to{opacity:1;transform:none}}
.card{background:linear-gradient(165deg,rgba(148,163,184,.16),rgba(100,116,139,.08) 45%,rgba(51,65,85,.1));border:1px solid rgba(226,232,240,.14);border-radius:18px;padding:22px;margin:18px 0;backdrop-filter:blur(16px);-webkit-backdrop-filter:blur(16px);box-shadow:0 14px 34px rgba(0,0,0,.5),inset 0 1px 0 rgba(255,255,255,.14),inset 0 -1px 0 rgba(0,0,0,.35);position:relative}
.card::before{content:'';position:absolute;inset:0;border-radius:18px;pointer-events:none;background:linear-gradient(115deg,transparent 30%,rgba(255,255,255,.09) 46%,rgba(255,255,255,.02) 52%,transparent 62%)}
.card h3{margin-bottom:10px;font-size:1rem}
.drop{border:2px dashed rgba(125,211,252,.5);border-radius:14px;padding:42px;text-align:center;cursor:pointer;background:rgba(56,189,248,.04);transition:.25s;color:#94a3b8}
.drop:hover{border-color:#7dd3fc;background:rgba(56,189,248,.08)}
.drop.dragover{border-color:#34d399;background:rgba(52,211,153,.12);color:#a7f3d0;box-shadow:0 0 24px rgba(52,211,153,.2)}
.bar{width:100%;height:14px;background:rgba(255,255,255,.07);border-radius:10px;overflow:hidden;margin:12px 0;box-shadow:inset 0 2px 4px rgba(0,0,0,.4)}
.bar>div{height:100%;width:0%;background:linear-gradient(90deg,#38bdf8,#6366f1,#a855f7);border-radius:10px;transition:width .3s;box-shadow:0 0 12px rgba(56,189,248,.5)}
.bar.green>div{background:linear-gradient(90deg,#10b981,#34d399)}
input[type=text],textarea{width:100%;padding:12px;border:1px solid rgba(255,255,255,.14);background:rgba(10,14,22,.7);color:#e2e8f0;border-radius:10px;font-family:ui-monospace,Menlo,monospace;font-size:.9rem;outline:none;transition:.2s}
input[type=text]:focus,textarea:focus{border-color:#38bdf8;box-shadow:0 0 0 3px rgba(56,189,248,.18)}
button.primary{background:linear-gradient(135deg,#38bdf8,#2563eb);color:#fff;border:0;padding:11px 22px;border-radius:10px;cursor:pointer;font-size:.92rem;font-weight:600;transition:.25s;box-shadow:0 4px 14px rgba(37,99,235,.4),inset 0 1px 0 rgba(255,255,255,.35),inset 0 -2px 4px rgba(0,0,0,.3)}
button.primary:hover{transform:translateY(-1px);box-shadow:0 6px 20px rgba(56,189,248,.5),inset 0 1px 0 rgba(255,255,255,.45)}
a.dl{display:inline-block;margin:6px 10px 6px 0;padding:10px 16px;background:linear-gradient(135deg,#10b981,#059669);color:#fff;border-radius:10px;text-decoration:none;font-size:.88rem;font-weight:600;transition:.2s;box-shadow:0 4px 14px rgba(16,185,129,.35),inset 0 1px 0 rgba(255,255,255,.3),inset 0 -2px 4px rgba(0,0,0,.25)}
a.dl:hover{transform:translateY(-1px);box-shadow:0 6px 18px rgba(16,185,129,.5)}
#s1,#fs,#gs,#ms,#extStatus,#extInfo{color:#94a3b8;font-size:.9rem}
#extInfo{background:rgba(255,255,255,.05);border:1px solid rgba(255,255,255,.08);border-radius:10px;padding:10px 12px;font-family:ui-monospace,Menlo,monospace}
</style>
</head>
<body>
<nav>
<button id="btn-upload" class="active" onclick="show('upload')">⚡ Upload ULP</button>
<button id="btn-extract" onclick="show('extract')">🔍 Extract Combo</button>
</nav>
<div id="page-upload" class="page active">
<h2>⚡ Upload ULP → Private Bucket</h2>
<div class="card">
<h3>From PC (direct, 5GB shards)</h3>
<div class="drop" id="drop">Drop .txt here or click<br><small style="color:#64748b">100GB supported</small><input type="file" id="file" accept=".txt" style="display:none"></div>
<div class="bar" id="p1" style="display:none"><div id="b1"></div></div>
<div id="s1"></div>
</div>
<div class="card">
<h3>From GoFile</h3>
<input type="text" id="gofile" placeholder="https://gofile.io/d/xxxx">
<button class="primary" onclick="fetchGofile()">Fetch GoFile → Bucket</button>
<div id="gs" style="margin-top:8px"></div><div class="bar" id="p3" style="display:none"><div id="b3"></div></div>
</div>
<div class="card">
<h3>From MediaFire</h3>
<input type="text" id="mediafire" placeholder="https://app.mediafire.com/folder/mdfd8hafxmmsy">
<button class="primary" onclick="fetchMediaFire()">Fetch MediaFire → Bucket</button>
<div id="ms" style="margin-top:8px"></div><div class="bar" id="p4" style="display:none"><div id="b4"></div></div>
</div>
<div class="card">
<h3>From Direct Link</h3>
<input type="text" id="url" placeholder="https://cdn2.filehost.org/download/xxxxx">
<button class="primary" onclick="fetchLink()">Fetch → Bucket</button>
<div id="fs" style="margin-top:8px"></div><div class="bar" id="p2" style="display:none"><div id="b2"></div></div>
</div>
</div>
<div id="page-extract" class="page">
<h2>🔍 Extract Combo from ULP</h2>
<div class="card">
<p style="color:#94a3b8;margin-bottom:10px">Enter keywords — <b style="color:#cbd5e1">one per line</b> (e.g. <code>binomo.com</code>, <code>hotmail.com</code>)</p>
<textarea id="kw" rows="6" placeholder="binomo&#10;hotmail.com&#10;Any keyword or site link"></textarea>
<button class="primary" onclick="doExtract()" style="margin-top:12px">Extract</button>
<div style="margin-top:12px;display:flex;align-items:center;gap:8px">
<button id="btn-adv" onclick="toggleAdv()" style="padding:8px 14px;border:1px solid rgba(226,232,240,.18);background:linear-gradient(165deg,rgba(203,213,225,.14),rgba(100,116,139,.04));color:#cbd5e1;border-radius:9px;cursor:pointer">⚙️ Advanced Settings</button>
<span id="selCount" style="color:#64748b;font-size:.85rem"></span>
</div>
<div id="advPanel" style="display:none;margin-top:12px;border:1px solid rgba(226,232,240,.14);border-radius:12px;padding:12px;background:rgba(8,12,20,.5)">
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:8px">
<b style="color:#94a3b8;font-size:.9rem">Select ULPs to extract (newest first)</b>
<div>
<button class="mini" onclick="selAll()">Select all</button>
<button class="mini" onclick="selNone()">Clear</button>
<button class="mini" onclick="sel24()">Last 24h</button>
</div>
</div>
<div id="ulplist" style="max-height:260px;overflow:auto;font-size:.85rem"></div>
<p style="color:#64748b;font-size:.8rem;margin-top:8px">Leave everything unchecked → scan the whole bucket.</p>
</div>
<div id="extStatus" style="margin-top:12px"></div>
<div class="bar" id="pE" style="display:none"><div id="bE"></div></div>
<div id="extInfo" style="margin-top:10px; font-family:monospace"></div>
<div id="extLinks" style="margin-top:14px"></div>
</div>
<style>.mini{padding:5px 10px;border:1px solid rgba(226,232,240,.16);background:rgba(148,163,184,.12);color:#cbd5e1;border-radius:7px;cursor:pointer;font-size:.78rem;margin-left:6px}.ulpitem{display:flex;align-items:center;gap:8px;padding:6px 8px;border-radius:7px;border:1px solid transparent;cursor:pointer}.ulpitem:hover{background:rgba(148,163,184,.1)}.ulpitem input{accent-color:#38bdf8}.ulpitem .nm{color:#cbd5e1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.ulpitem .sz{color:#64748b;font-size:.75rem;flex:none}</style>
</div>
<pre id="log"></pre>
<script>
function show(p){document.querySelectorAll('.page').forEach(e=>e.classList.remove('active'));document.querySelectorAll('nav button').forEach(e=>e.classList.remove('active'));document.getElementById('page-'+p).classList.add('active');document.getElementById('btn-'+p).classList.add('active');}
const drop=document.getElementById('drop'), fileInput=document.getElementById('file'), b1=document.getElementById('b1'), p1=document.getElementById('p1'), s1=document.getElementById('s1');
drop.onclick=()=>fileInput.click();
drop.ondragover=e=>{e.preventDefault();drop.classList.add('dragover')};
drop.ondragleave=()=>drop.classList.remove('dragover');
drop.ondrop=e=>{e.preventDefault();drop.classList.remove('dragover');handle(e.dataTransfer.files[0])};
fileInput.onchange=e=>handle(e.target.files[0]);
async function handle(file){
if(!file)return;
p1.style.display='block'; s1.textContent=`Uploading ${file.name} (${(file.size/1024/1024/1024).toFixed(2)} GB)...`;
const fd=new FormData(); fd.append('file', file);
const xhr=new XMLHttpRequest();
xhr.upload.onprogress=e=>{ if(e.lengthComputable){ const pct=e.loaded/e.total*100; b1.style.width=pct+'%'; s1.textContent=`Uploading ${file.name} ${pct.toFixed(1)}%`; } };
xhr.onload=()=>{ s1.textContent=xhr.status==200?'✅ Done — in bucket':'❌ Failed'; b1.style.background='#10b981'; document.getElementById('log').textContent+=xhr.responseText+'\\n'; };
xhr.open('POST','/upload'); xhr.send(fd);
}
async function fetchLink(){
const url=document.getElementById('url').value.trim(); if(!url){alert('Paste URL');return;}
document.getElementById('fs').textContent='Fetching...'; document.getElementById('p2').style.display='block';
const r=await fetch('/fetch',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({url})});
const j=await r.json(); if(j.job_id) poll(j.job_id,'fetch'); else document.getElementById('fs').textContent=JSON.stringify(j);
}
async function fetchGofile(){
const url=document.getElementById('gofile').value.trim(); if(!url){alert('Paste GoFile URL');return;}
document.getElementById('gs').textContent='Resolving GoFile...'; document.getElementById('p3').style.display='block';
const r=await fetch('/fetch/gofile',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({url})});
const j=await r.json(); if(j.job_id) poll(j.job_id,'gofile'); else document.getElementById('gs').textContent=JSON.stringify(j);
}
async function poll(job_id,type){
const bar=document.getElementById(type=='gofile'?'b3':'b2'), st=document.getElementById(type=='gofile'?'gs':'fs');
const iv=setInterval(async()=>{
const r=await fetch('/fetch/progress?job_id='+job_id); const j=await r.json();
st.textContent=j.status+ (j.pct?` ${j.pct}%`:''); if(j.pct) bar.style.width=j.pct+'%';
if(j.status.includes('Done')||j.status.includes('Failed')) clearInterval(iv);
},1000);
}
async function doExtract(){
const raw=document.getElementById('kw').value.trim(); if(!raw){alert('Enter keywords, one per line');return;}
const targets=raw.split(/\\n/).map(s=>s.trim()).filter(Boolean);
const files=Array.from(document.querySelectorAll('#ulplist input:checked')).map(c=>c.value);
document.getElementById('extStatus').textContent='Starting extraction'+(files.length?` (${files.length} ULP${files.length>1?'s':''})`:' (whole bucket)')+ '...';
document.getElementById('pE').style.display='block';
document.getElementById('extLinks').innerHTML='';
const body={targets};
if(files.length) body.files=files;
const r=await fetch('/extract',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(body)});
const j=await r.json(); if(!j.job_id){ document.getElementById('extStatus').textContent=JSON.stringify(j); return; }
pollExtract(j.job_id);
}
let ulpCache=[];
async function toggleAdv(){
const p=document.getElementById('advPanel');
if(p.style.display==='none'){
p.style.display='block';
if(!ulpCache.length){
const ulp=document.getElementById('ulplist'); ulp.innerHTML='<span style="color:#64748b">Loading bucket files…</span>';
try{
const r=await fetch('/bucket/files'); const j=await r.json();
ulpCache=j.files||[];
}catch(e){ulpCache=[];}
}
renderUlp();
} else { p.style.display='none'; }
}
function renderUlp(){
const ulp=document.getElementById('ulplist');
if(!ulpCache.length){ ulp.innerHTML='<span style="color:#64748b">No ULPs in bucket.</span>'; updateSelCount(); return; }
const fmt=s=>s>=1024**3?(s/1024**3).toFixed(1)+' GB':(s/1024**2|0)+' MB';
ulp.innerHTML=ulpCache.map(f=>`<label class="ulpitem" title="${f.name} (${fmt(f.size)}${f.uploaded_at?', '+f.uploaded_at.slice(0,16):''})" style="display:flex;align-items:center;gap:8px">
<input type="checkbox" value="${f.name.replace(/"/g,'&quot;')}" data-ts="${f.ts||0}">
<span class="nm">${f.name}</span>
<span class="sz">${fmt(f.size)}</span></label>`).join('');
ulp.onchange=()=>updateSelCount();
updateSelCount();
}
function updateSelCount(){
const n=document.querySelectorAll('#ulplist input:checked').length;
document.getElementById('selCount').textContent=n?`${n} selected`:'All ULPs (whole bucket)';
}
function selAll(){ document.querySelectorAll('#ulplist input').forEach(c=>c.checked=true); updateSelCount(); }
function selNone(){ document.querySelectorAll('#ulplist input').forEach(c=>c.checked=false); updateSelCount(); }
function sel24(){
const cutoff=Date.now()/1000-86400;
document.querySelectorAll('#ulplist input').forEach(c=>{
const ts=parseFloat(c.dataset.ts||'0');
c.checked=ts>0 && ts>=cutoff;
});
updateSelCount();
}
async function fetchMediaFire(){
const url=document.getElementById('mediafire').value.trim(); if(!url){alert('Paste MediaFire link');return;}
document.getElementById('ms').textContent='Resolving MediaFire...'; document.getElementById('p4').style.display='block';
const r=await fetch('/mediafire',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({url})});
const j=await r.json(); if(j.job_id) pollMf(j.job_id); else document.getElementById('ms').textContent=JSON.stringify(j);
}
async function pollMf(job_id){
const bar=document.getElementById('b4'), st=document.getElementById('ms');
const iv=setInterval(async()=>{
const r=await fetch('/fetch/progress?job_id='+job_id); const j=await r.json();
st.textContent=j.status+(j.pct?` ${j.pct}%`:''); if(j.pct) bar.style.width=j.pct+'%';
if(j.status.includes('Done')||j.status.includes('Failed')) clearInterval(iv);
},1000);
}
async function pollExtract(job_id){
const bar=document.getElementById('bE'), st=document.getElementById('extStatus'), info=document.getElementById('extInfo'), links=document.getElementById('extLinks');
const iv=setInterval(async()=>{
const r=await fetch('/extract/progress?job_id='+job_id); const j=await r.json();
st.textContent=j.status||''; if(j.pct) bar.style.width=j.pct+'%';
if(j.total_gb) info.textContent=`Scanned: ${j.scanned_gb||0}GB / ${j.total_gb}GB • ${j.scanned_lines||0} lines • Shards: ${j.shards} • Found: ${j.found||0} • ${j.speed||''}`;
if(j.done){
clearInterval(iv);
if(j.files && j.files.length){
links.innerHTML=j.files.map(f=>{
const href=f.download_url||`/extract/download?file=${encodeURIComponent(f.name)}`;
return `<a class="dl" href="${href}" download="${f.name}">⬇ ${f.name} (${f.lines} lines)</a>`;
}).join('');
}
if(j.error) st.textContent+=' ❌ '+j.error;
}
},1000);
}
</script>
</body>
</html>
"""
@app.get("/", response_class=HTMLResponse)
async def index():
return HTML
@app.post("/upload")
async def upload(file: UploadFile = File(...)):
tmp_path = f"/tmp/{uuid.uuid4().hex}_{file.filename}"
try:
with open(tmp_path, "wb") as out:
while True:
chunk = await file.read(8*1024*1024)
if not chunk: break
out.write(chunk)
size = os.path.getsize(tmp_path)
shards=[]
if size <= SHARD_SIZE:
batch_bucket_files(BUCKET_ID, add=[(tmp_path, file.filename)])
shards=[file.filename]
else:
idx=0
with open(tmp_path, "rb") as fh:
while True:
chunk_path=f"{tmp_path}.part{idx:03d}"
with open(chunk_path,"wb") as out:
written=0
while written < SHARD_SIZE:
data=fh.read(4*1024*1024)
if not data: break
out.write(data)
written+=len(data)
if written==0:
try: os.remove(chunk_path)
except: pass
break
if os.path.getsize(chunk_path)==0: break
hf_name=f"{os.path.splitext(file.filename)[0]}_part{idx:03d}.txt"
batch_bucket_files(BUCKET_ID, add=[(chunk_path, hf_name)])
shards.append(hf_name)
try: os.remove(chunk_path)
except: pass
idx+=1
if fh.tell()>=size: break
try: os.remove(tmp_path)
except: pass
return JSONResponse({"status":"ok","file":file.filename,"shards":shards,"size":size,"bucket":BUCKET_ID})
except Exception as e:
traceback.print_exc()
try: os.remove(tmp_path)
except: pass
return JSONResponse({"status":"error","error":str(e)}, status_code=500)
def do_fetch(job_id, url):
jobs[job_id]={"status":"Resolving...","pct":0}
try:
tmp_base=f"/tmp/fetch_{job_id}"
os.makedirs(tmp_base, exist_ok=True)
orig_url=url
if "mediafire.com/file/" in url:
try:
import requests
r=requests.get(url, timeout=30, headers={"User-Agent":"Mozilla/5.0"})
m=re.search(r'https://download\d+\.mediafire\.com/[^"]+', r.text)
if m:
url=m.group(0)
jobs[job_id]={"status":"Resolved Mediafire direct link","pct":2}
except Exception as e:
jobs[job_id]={"status":f"Mediafire resolve failed: {e}","pct":0}
fname=url.split("/")[-1].split("?")[0] or orig_url.split("/")[-1].split("?")[0] or "fetched.txt"
fname=re.sub(r'[^A-Za-z0-9._ -]', '_', fname)[:80] or "fetched.txt"
if not fname.endswith(".txt"): fname+=".txt"
try:
import requests
h=requests.head(url, allow_redirects=True, timeout=10)
cd=h.headers.get("content-disposition","")
if "filename=" in cd:
fname=re.sub(r'[^A-Za-z0-9._ -]', '_', cd.split("filename=")[-1].strip('\" ;'))[:80]
if not fname.endswith(".txt"): fname+=".txt"
except: pass
tmp_path=os.path.join(tmp_base, fname)
jobs[job_id]={"status":f"Downloading {fname}...","pct":5}
try:
cmd=["aria2c","-x","16","-s","16","-k","1M","-o",tmp_path,url,"--allow-overwrite=true","--auto-file-renaming=false","--console-log-level=warn"]
proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except FileNotFoundError:
cmd=["curl","-L","-o",tmp_path,url]
proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
while proc.poll() is None:
try:
cur=os.path.getsize(tmp_path) if os.path.exists(tmp_path) else 0
jobs[job_id]={"status":f"Downloading {fname} {cur//1024//1024}MB","pct":10,"speed":f"{cur//1024}KB"}
except: pass
time.sleep(1)
if proc.returncode!=0 or not os.path.exists(tmp_path) or os.path.getsize(tmp_path)==0:
jobs[job_id]={"status":f"Failed: {proc.returncode}","pct":0}
return
size=os.path.getsize(tmp_path)
jobs[job_id]={"status":f"Downloaded {size//1024//1024}MB, uploading to bucket...","pct":50}
if size <= SHARD_SIZE:
batch_bucket_files(BUCKET_ID, add=[(tmp_path, fname)])
jobs[job_id]={"status":f"Done — {fname} ({size//1024//1024}MB) → bucket","pct":100}
else:
idx=0
shards=[]
with open(tmp_path,"rb") as fh:
while True:
chunk_path=f"{tmp_path}.part{idx:03d}"
with open(chunk_path,"wb") as out:
written=0
while written < SHARD_SIZE:
data=fh.read(4*1024*1024)
if not data: break
out.write(data)
written+=len(data)
if written==0:
try: os.remove(chunk_path)
except: pass
break
if not os.path.exists(chunk_path) or os.path.getsize(chunk_path)==0: break
hf_name=f"{os.path.splitext(fname)[0]}_part{idx:03d}.txt"
jobs[job_id]={"status":f"Uploading shard {idx+1} {written//1024//1024}MB...","pct":50+idx*10}
batch_bucket_files(BUCKET_ID, add=[(chunk_path, hf_name)])
shards.append(hf_name)
try: os.remove(chunk_path)
except: pass
idx+=1
if fh.tell()>=size: break
jobs[job_id]={"status":f"Done — {len(shards)} shards of {fname} → bucket","pct":100}
try: os.remove(tmp_path)
except: pass
try: shutil.rmtree(tmp_base)
except: pass
except Exception as e:
traceback.print_exc()
jobs[job_id]={"status":f"Failed: {e}","pct":0}
def do_gofile(job_id, url):
import hashlib, json as js
jobs[job_id]={"status":"Resolving GoFile...","pct":2}
try:
m=re.search(r"gofile\.io/d/([A-Za-z0-9-]+)", url)
if not m:
jobs[job_id]={"status":"Invalid GoFile URL","pct":0}; return
content_id=m.group(1)
GOFILE_UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
GOFILE_LANG="en-US"
GOFILE_SALT="12af056dacea0b"
def website_token(acc, offset=0):
import hashlib as hl, time as tm
window=int(tm.time()//14400)+offset
raw=f"{GOFILE_UA}::{GOFILE_LANG}::{acc}::{window}::{GOFILE_SALT}"
return hl.sha256(raw.encode()).hexdigest()
out=subprocess.run(["curl","-sS","-X","POST","https://api.gofile.io/accounts","-H",f"User-Agent: {GOFILE_UA}","-H","Origin: https://gofile.io","-d",""], capture_output=True, text=True, timeout=30)
token=(js.loads(out.stdout).get("data",{}) or {}).get("token")
if not token:
jobs[job_id]={"status":"No guest token","pct":0}; return
jobs[job_id]={"status":"Guest token ok, resolving...","pct":10}
def contents(acc, off, cid):
cmd=["curl","-sS","-G",f"https://api.gofile.io/contents/{cid}","-H",f"Authorization: Bearer {acc}","-H",f"X-Website-Token: {website_token(acc,off)}","-H",f"X-BL: {GOFILE_LANG}","-H",f"User-Agent: {GOFILE_UA}","-H","Accept: */*","-H","Origin: https://gofile.io","-H","Referer: https://gofile.io/","--data-urlencode","contentFilter=","--data-urlencode","page=1","--data-urlencode","pageSize=1000","--data-urlencode","sortField=createTime","--data-urlencode","sortDirection=-1"]
out=subprocess.run(cmd, capture_output=True, text=True, timeout=30)
return js.loads(out.stdout)
def collect(acc, off, cid):
out=contents(acc, off, cid)
st=out.get("status")
if st=="error-notPremium" and off==0:
out=contents(acc,-1,cid); st=out.get("status")
if st!="ok": return None, f"gofile: {st}"
d=out.get("data",{}) or {}
if d.get("type")=="file" or d.get("link"):
return [(d.get("name") or "gofile_file.txt", int(d.get("size") or 0), d.get("link") or "")], None
entries=[]
for child_id,c in (d.get("children") or {}).items():
if c.get("type")=="folder":
sub,err=collect(acc,off,child_id)
if err: return None, err
entries.extend(sub)
elif c.get("type")=="file":
entries.append((c.get("name") or "gofile_file.txt", int(c.get("size") or 0), c.get("link") or ""))
return entries, None
files,err=None,None
for off in (0,-1):
files,err=collect(token,off,content_id)
if files is not None and not err: break
if not files:
jobs[job_id]={"status":f"Failed: {err or 'no files'}","pct":0}; return
files=[(n,s,u) for n,s,u in files if u]
total=len(files)
jobs[job_id]={"status":f"Found {total} file(s), downloading...","pct":15}
for idx,(fname_raw,fsize_b,dl_url) in enumerate(files,1):
fname=re.sub(r'[^A-Za-z0-9._ -]', '_', fname_raw)[:80] or f"gofile_{idx}.txt"
tmp_base=f"/tmp/gofile_{job_id}_{idx}"
os.makedirs(tmp_base, exist_ok=True)
tmp_path=os.path.join(tmp_base, fname)
cmd=["curl","-sS","-L","-o",tmp_path,dl_url,"-H",f"Cookie: accountToken={token}","-H",f"User-Agent: {GOFILE_UA}","-H","Referer: https://gofile.io/"]
proc=subprocess.Popen(cmd)
while proc.poll() is None:
try:
cur=os.path.getsize(tmp_path) if os.path.exists(tmp_path) else 0
pct=15+ int((idx-1)/total*70 + (cur/max(1,fsize_b))*70/total) if fsize_b else 15+int((idx-1)/total*70)
jobs[job_id]={"status":f"Downloading {idx}/{total} {fname} {cur//1024//1024}MB/{fsize_b//1024//1024}MB","pct":pct}
except: pass
time.sleep(1)
if proc.returncode!=0 or not os.path.exists(tmp_path) or os.path.getsize(tmp_path)==0:
jobs[job_id]={"status":f"Failed {fname}","pct":0}; continue
size=os.path.getsize(tmp_path)
if size <= SHARD_SIZE:
batch_bucket_files(BUCKET_ID, add=[(tmp_path, fname)])
else:
with open(tmp_path,"rb") as fh:
part=0
while True:
chunk_path=f"{tmp_path}.part{part:03d}"
with open(chunk_path,"wb") as out:
written=0
while written < SHARD_SIZE:
data=fh.read(4*1024*1024)
if not data: break
out.write(data)
written+=len(data)
if written==0:
try: os.remove(chunk_path)
except: pass
break
if not os.path.exists(chunk_path) or os.path.getsize(chunk_path)==0: break
hf_name=f"{os.path.splitext(fname)[0]}_part{part:03d}.txt"
batch_bucket_files(BUCKET_ID, add=[(chunk_path, hf_name)])
try: os.remove(chunk_path)
except: pass
part+=1
if fh.tell()>=size: break
try: os.remove(tmp_path)
except: pass
try: shutil.rmtree(tmp_base)
except: pass
jobs[job_id]={"status":f"Done — {total} file(s) → bucket","pct":100}
except Exception as e:
traceback.print_exc()
jobs[job_id]={"status":f"Failed: {e}","pct":0}
# ---- MediaFire ----
MF_UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
def _mf_list_folder(folder_key):
files=[]; chunk=1
while True:
u=(f"https://www.mediafire.com/api/1.4/folder/get_content.php?r=utga"
f"&content_type=files&filter=all&order_by=name&order_direction=asc"
f"&chunk={chunk}&version=1.5&folder_key={folder_key}&response_format=json")
import requests
r=requests.get(u, headers={"User-Agent":MF_UA}, timeout=30)
fc=r.json().get("response",{}).get("folder_content",{})
for f in fc.get("files",[]):
files.append({"quickkey":f.get("quickkey",""),
"filename":f.get("filename","unknown.txt"),
"size":int(f.get("size",0) or 0),
"normal":(f.get("links",{}) or {}).get("normal_download","")})
if str(fc.get("more_chunks","no")).lower()!="yes" or chunk>200: break
chunk+=1
return files
def _mf_list_subfolders(folder_key):
u=(f"https://www.mediafire.com/api/1.4/folder/get_content.php?r=utga"
f"&content_type=folders&filter=all&chunk=1&version=1.5"
f"&folder_key={folder_key}&response_format=json")
import requests
try:
r=requests.get(u, headers={"User-Agent":MF_UA}, timeout=30)
return r.json().get("response",{}).get("folder_content",{}).get("folders",[]) or []
except Exception:
return []
def _mf_collect_folder(folder_key, depth=0):
files=_mf_list_folder(folder_key)
if depth<5:
for sub in _mf_list_subfolders(folder_key):
files+=_mf_collect_folder(sub.get("folderkey",""), depth+1)
return files
def _mf_resolve_file(quickkey, filename):
import requests
page=f"https://www.mediafire.com/file/{quickkey}/{filename}/file"
try:
r=requests.get(page, headers={"User-Agent":MF_UA}, timeout=30)
for m in re.finditer(r'https://download\d+\.mediafire\.com/[^"\'\s<]+', r.text):
return m.group(0).replace("&amp;","&")
except Exception:
pass
return None
def _mf_dl_and_upload(job_id, dl_url, fname, base_status):
import requests
tmp_base=f"/tmp/mf_{job_id}"
os.makedirs(tmp_base, exist_ok=True)
tmp_path=os.path.join(tmp_base, fname)
try:
r=requests.get(dl_url, headers={"User-Agent":MF_UA}, stream=True, timeout=60, allow_redirects=True)
if r.status_code!=200:
jobs[job_id]["status"]=f"{base_status} → HTTP {r.status_code} {r.headers.get('content-type','')[:40]} len={r.headers.get('content-length','?')}"
try: r.close()
except: pass
return False
clen=int(r.headers.get("content-length",0) or 0)
written=0
with open(tmp_path,"wb") as out:
for chunk in r.iter_content(chunk_size=8*1024*1024):
if not chunk: continue
out.write(chunk); written+=len(chunk)
if written%(64*1024*1024)==0:
jobs[job_id]=dict(jobs.get(job_id,{}))
jobs[job_id]["status"]=f"{base_status} {written//1024//1024}MB"+(f"/{clen//1024//1024}MB" if clen else "")
try: r.close()
except: pass
if written==0:
jobs[job_id]["status"]=f"{base_status} → empty body (0 bytes)"
return False
except Exception as e:
jobs[job_id]["status"]=f"{base_status} → requests err: {str(e)[:160]}"
try: shutil.rmtree(tmp_base)
except: pass
return False
size=os.path.getsize(tmp_path)
jobs[job_id]["status"]=f"{base_status} → uploading {size//1024//1024}MB to bucket..."
if size<=SHARD_SIZE:
batch_bucket_files(BUCKET_ID, add=[(tmp_path, fname)])
else:
with open(tmp_path,"rb") as fh:
part=0
while True:
chunk_path=f"{tmp_path}.part{part:03d}"
with open(chunk_path,"wb") as out:
written=0
while written<SHARD_SIZE:
data=fh.read(4*1024*1024)
if not data: break
out.write(data); written+=len(data)
if written==0:
try: os.remove(chunk_path)
except: pass
break
if not os.path.exists(chunk_path) or os.path.getsize(chunk_path)==0: break
hf_name=f"{os.path.splitext(fname)[0]}_part{part:03d}.txt"
batch_bucket_files(BUCKET_ID, add=[(chunk_path, hf_name)])
try: os.remove(chunk_path)
except: pass
part+=1
if fh.tell()>=size: break
try: os.remove(tmp_path)
except: pass
try: shutil.rmtree(tmp_base)
except: pass
return True
def do_mediafire(job_id, url):
import requests
jobs[job_id]={"status":"Resolving MediaFire...","pct":2}
try:
folder_m=re.search(r'mediafire\.com/[^/]+/folder/([A-Za-z0-9]+)', url)
if not folder_m:
folder_m=re.search(r'mediafire\.com/folder/([A-Za-z0-9]+)', url)
file_m=re.search(r'mediafire\.com/(?:file_premium|file)/([A-Za-z0-9]+)/([^/?&]+)', url)
if folder_m and (not file_m or "mediafire.com/folder/" in url):
folder_key=folder_m.group(1)
jobs[job_id]={"status":"Listing folder files...","pct":4}
all_files=_mf_collect_folder(folder_key)
if not all_files:
jobs[job_id]={"status":"No files found in folder","pct":0}; return
total=len(all_files)
total_gb=sum(f["size"] for f in all_files)/1024**3
jobs[job_id].update({"status":f"Found {total} files ({total_gb:.1f}GB), starting download...","pct":6,"total":total})
ok=0
for idx,f in enumerate(all_files,1):
fname=re.sub(r'[^A-Za-z0-9._ -]', '_', f["filename"] or f"mf_{idx}.txt")[:80] or f"mf_{idx}.txt"
if not fname.endswith(".txt"): fname+=".txt"
dl_url=None
if f.get("normal"):
try:
r=requests.get(f["normal"], headers={"User-Agent":MF_UA}, timeout=30, allow_redirects=True)
m=re.search(r'https://download\d+\.mediafire\.com/[^"\'\s<]+', r.text)
if m: dl_url=m.group(0).replace("&amp;","&")
except Exception:
pass
if not dl_url:
dl_url=_mf_resolve_file(f["quickkey"], f["filename"])
if not dl_url:
jobs[job_id].update({"status":f"Skip {idx}/{total}: can't resolve {fname}","pct":6+int((idx-1)/total*85)})
continue
jobs[job_id].update({"status":f"Downloading {idx}/{total}: {fname}","pct":6+int((idx-1)/total*85)})
if _mf_dl_and_upload(job_id, dl_url, fname, f"Downloading {idx}/{total}"):
ok+=1
jobs[job_id].update({"pct":6+int(idx/total*85)})
jobs[job_id]={"status":f"Done — {ok}/{total} file(s) → bucket","pct":100,"total":total}
elif file_m:
quickkey=file_m.group(1)
fname=re.sub(r'[^A-Za-z0-9._ -]', '_', unquote(file_m.group(2))) or "mf_file.txt"
if not fname.endswith(".txt"): fname+=".txt"
dl_url=_mf_resolve_file(quickkey, file_m.group(2))
if not dl_url:
try:
r=requests.get(url, headers={"User-Agent":MF_UA}, timeout=30)
m=re.search(r'https://download\d+\.mediafire\.com/[^"\'\s<]+', r.text)
if m: dl_url=m.group(0).replace("&amp;","&")
except Exception:
pass
if not dl_url:
jobs[job_id]={"status":"Failed: can't resolve MediaFire file","pct":0}; return
jobs[job_id]={"status":f"Downloading {fname}...","pct":8}
ok=_mf_dl_and_upload(job_id, dl_url, fname, f"Downloading {fname}")
if ok:
jobs[job_id]={"status":f"Done — {fname} → bucket","pct":100}
else:
jobs[job_id]={"status":"Invalid MediaFire URL (use /folder/ or /file/ link)","pct":0}
except Exception as e:
traceback.print_exc()
jobs[job_id]={"status":f"Failed: {e}","pct":0}
# ---- Extract ----
import hashlib
def _hash_line(line: str) -> int:
import hashlib as hl
return int.from_bytes(hl.blake2b(line.encode("utf-8", "ignore"), digest_size=8).digest(), "big")
def list_bucket_files():
from datetime import datetime, timezone
infos = hffs.ls(f"buckets/{BUCKET_ID}", detail=True, refresh=True)
out=[]
for i in infos:
if not isinstance(i, dict): continue
nm=str(i.get("name",""))
if not nm.lower().endswith(".txt"): continue
base=nm.split(f"buckets/{BUCKET_ID}/")[-1] if f"buckets/{BUCKET_ID}/" in nm else nm
uploaded_at=str(i.get("uploaded_at") or i.get("mtime") or "")
ts=0
if uploaded_at:
try:
dt=datetime.fromisoformat(uploaded_at.replace("Z","+00:00"))
ts=dt.timestamp()
except Exception:
try:
dt=datetime.strptime(uploaded_at[:19], "%Y-%m-%d %H:%M:%S")
ts=dt.replace(tzinfo=timezone.utc).timestamp()
except Exception:
ts=0
out.append({"name":base,
"size":int(i.get("size",0) or 0),
"uploaded_at":uploaded_at,
"ts":ts})
out.sort(key=lambda x: x["uploaded_at"], reverse=True)
return out
def do_extract(job_id, targets, selected=None):
jobs[job_id]={"status":"Listing bucket...","pct":5,"found":0,"shards":0,"total_gb":"0"}
try:
# list shards
if selected:
sel=set(selected)
infos = hffs.ls(f"buckets/{BUCKET_ID}", detail=True, refresh=True)
all_shards=[]
for i in infos:
if not isinstance(i, dict): continue
nm=str(i.get("name",""))
if not nm.lower().endswith(".txt"): continue
base=nm.split(f"buckets/{BUCKET_ID}/")[-1] if f"buckets/{BUCKET_ID}/" in nm else nm
if base in sel:
all_shards.append(nm)
else:
paths = hffs.ls(f"buckets/{BUCKET_ID}", detail=False, refresh=True)
all_shards = [p for p in paths if p.lower().endswith(".txt")]
shards = all_shards
per_gb=dict()
try:
for i in hffs.ls(f"buckets/{BUCKET_ID}", detail=True, refresh=True):
if not isinstance(i, dict): continue
nm=str(i.get("name",""))
if not nm.lower().endswith(".txt"): continue
base=nm.split(f"buckets/{BUCKET_ID}/")[-1] if f"buckets/{BUCKET_ID}/" in nm else nm
per_gb[nm]=int(i.get("size",0) or 0)/1024**3
except Exception:
pass
total_gb = sum(per_gb.get(p,2.2) for p in shards) if shards else 0
jobs[job_id].update({"total_gb":f"{total_gb:.1f}","shards":len(shards)})
if not shards:
jobs[job_id]={"status":"No txt in bucket (or none selected)","pct":0,"done":True}; return
out_dir = f"/tmp/extract_{job_id}"
os.makedirs(out_dir, exist_ok=True)
needles = [t.lower() for t in targets]
pat_path = os.path.join(out_dir, "patterns.txt")
with open(pat_path,"w") as pf:
for n in needles:
pf.write(n+"\n")
counts={t:0 for t in targets}
seens={t:set() for t in targets}
outs={t: open(os.path.join(out_dir, f"{t.replace('.','_')}.txt"),"w",encoding="utf-8",errors="ignore") for t in targets}
total_scanned=0
done=0
def process_shard(hf_path):
tmp_dl = os.path.join(out_dir, f"dl_{os.path.basename(hf_path)}")
try:
from huggingface_hub import download_bucket_files
rel = hf_path.split(f"buckets/{BUCKET_ID}/")[-1]
download_bucket_files(bucket_id=BUCKET_ID, files=[(rel, tmp_dl)])
local_path = tmp_dl
except Exception:
with hffs.open(hf_path,"r") as src, open(tmp_dl,"w",encoding="utf-8",errors="ignore") as dst:
shutil.copyfileobj(src, dst)
local_path = tmp_dl
cmd=["grep","-a","-i","-F","-f",pat_path,"--",local_path]
# use ripgrep if available
try:
import shutil as sh
if sh.which("rg"):
cmd=["rg","-a","-i","-F","-f",pat_path,"--",local_path]
except: pass
proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, errors="ignore", bufsize=1024*1024)
local_counts={t:0 for t in targets}
# accurate parse like bot
import re as re2
_SCHEME_RE = re2.compile(r"^[a-zA-Z][a-zA-Z0-9+.-]*://")
_DOMAIN_RE = re2.compile(r"(?:www\.)?([a-zA-Z0-9](?:[a-zA-Z0-9._-]*[a-zA-Z0-9])?\.[a-zA-Z]{2,})")
def parse_line(l):
l=l.strip()
if not l: return None
l=_SCHEME_RE.sub("", l)
idx=-1
for d in "|;,\t:":
i=l.find(d)
if i!=-1 and (idx==-1 or i<idx):
idx=i
if idx==-1:
m=_DOMAIN_RE.search(l)
site=m.group(1).lower() if m else l.lower().rstrip("/")
return site, []
site=l[:idx]
m=_DOMAIN_RE.search(site)
site=m.group(1).lower() if m else site.lower().rstrip("/")
rest=l[idx+1:]
for d in "|;,\t":
rest=rest.replace(d,":")
vals=[v for v in rest.split(":") if v]
return site, vals
def site_matches(site,tgt):
t=[x for x in tgt.strip(".").split(".") if x]
if not t: return False
labels=site.split(".")
if any(labels[i:i+len(t)]==t for i in range(len(labels)-len(t)+1)):
return True
h_labels=site.replace("-",".").split(".")
h_labels=[x for x in h_labels if x]
if any(h_labels[i:i+len(t)]==t for i in range(len(h_labels)-len(t)+1)):
return True
return False
for line in proc.stdout:
r=parse_line(line)
if not r or not r[1]: continue
site, vals = r
joined=":".join(vals)
for tgt, needle in zip(targets, needles):
is_domain="." in tgt
if is_domain:
if not site_matches(site,tgt): continue
else:
if needle not in line.lower(): continue
h=_hash_line(joined)
if h not in seens[tgt]:
seens[tgt].add(h)
outs[tgt].write(joined+"\n")
local_counts[tgt]+=1
proc.stdout.close(); proc.wait()
try: os.remove(local_path)
except: pass
return local_counts
workers = 16 if len(shards) > 16 else 8
done_set=set()
with ThreadPoolExecutor(max_workers=workers) as ex:
futs={ex.submit(process_shard, p): p for p in shards}
for fut in as_completed(futs):
p=futs[fut]
done+=1
done_set.add(p)
try:
res=fut.result()
for tgt in targets:
counts[tgt]+=res.get(tgt,0)
except Exception as e:
traceback.print_exc()
pct=10+ int(done/len(shards)*80)
scanned_gb = sum(per_gb.get(dp,2.2) for dp in done_set)
scanned_lines_est = done * 5000000
jobs[job_id].update({"status":f"Scanning {done}/{len(shards)}","pct":pct,"found":sum(counts.values()),"scanned_gb":f"{scanned_gb:.1f}","scanned_lines":f"{scanned_lines_est//1000000}M", "speed":"~48 MB/s"})
for fh in outs.values(): fh.close()
files=[]
for tgt in targets:
path=os.path.join(out_dir, f"{tgt.replace('.','_')}.txt")
if os.path.exists(path) and os.path.getsize(path)>0:
name=f"{tgt}.txt"
safe=re.sub(r'[^A-Za-z0-9._-]', '_', name)
dl={}
final=None
# 1) persist on the Space's stable disk (survives restarts) + /tmp fallback
try:
pd="/data/extracts"; os.makedirs(pd, exist_ok=True)
pd_path=os.path.join(pd, safe)
shutil.copy(path, pd_path)
final=pd_path
dl["download_url"]=f"/extract/download?file={safe}"
except Exception as e:
print("persist fail:", e, flush=True)
if final is None or (os.path.exists(final) and os.path.getsize(final)==0):
try:
td="/tmp/extracts"; os.makedirs(td, exist_ok=True)
tp=os.path.join(td, safe)
shutil.copy(path, tp); final=tp
dl["download_url"]=f"/extract/download?file={safe}"
except Exception as e:
print("tmp fail:", e, flush=True)
if final is None:
final=f"/tmp/extract_{job_id}_{tgt.replace('.','_')}.txt"
shutil.copy(path, final)
# 2) push into the Space repo files/ folder — DISABLED: causes Space rebuild which wipes jobs
# if you want repo backup, re-enable below (will trigger a restart ~30s later)
"""
try:
from huggingface_hub import HfApi as _HfApi
token=os.environ.get("HF_TOKEN") or os.environ.get("HF_TOKEN_WRITE")
if token and os.environ.get("HF_SPACE_ID"):
api=_HfApi(token=token)
api.upload_file(path_or_fileobj=path,
path_in_repo=f"files/extracts/{safe}",
repo_id=os.environ["HF_SPACE_ID"], repo_type="space")
dl["hf_url"]=(f"https://huggingface.co/spaces/{os.environ['HF_SPACE_ID']}/"
f"resolve/main/files/extracts/{safe}?download=true")
except Exception as e:
print("repo push fail:", e, flush=True)
"""
files.append({"name":name,"lines":counts[tgt],"path":final, **dl})
else:
try: os.remove(path)
except: pass
try: os.remove(pat_path)
except: pass
try: shutil.rmtree(out_dir)
except: pass
jobs[job_id].update({"status":f"Done — {sum(counts.values())} lines","pct":100,"done":True,"files":files,"found":sum(counts.values())})
except Exception as e:
traceback.print_exc()
jobs[job_id]={"status":f"Failed: {e}","pct":0,"done":True,"error":str(e)}
@app.post("/extract")
async def extract(request: Request):
data=await request.json()
targets=data.get("targets",[])
if isinstance(targets, str):
targets=[t.strip() for t in targets.split("\n") if t.strip()]
targets=[t.strip() for t in targets if t.strip()]
if not targets:
return JSONResponse({"error":"Enter at least one keyword, one per line"}, status_code=400)
if len(targets)>20:
return JSONResponse({"error":"Max 20 keywords"}, status_code=400)
selected=data.get("files")
if isinstance(selected, str):
selected=[s.strip() for s in selected.split("\n") if s.strip()]
if isinstance(selected, list):
selected=[s for s in selected if s and isinstance(s,str)]
if not selected: selected=None
job_id=uuid.uuid4().hex[:8]
jobs[job_id]={"status":"Queued...","pct":0,"done":False}
import threading
threading.Thread(target=do_extract, args=(job_id, targets, selected), daemon=True).start()
return {"job_id":job_id}
@app.get("/bucket/files")
async def bucket_files_list():
try:
return {"files": list_bucket_files()}
except Exception as e:
return {"error":str(e), "files":[]}
@app.get("/extract/progress")
async def extract_progress(job_id: str):
j=jobs.get(job_id, {"status":"Unknown","pct":0})
out={k:v for k,v in j.items() if not k.startswith("file_")}
return out
@app.get("/extract/download")
async def extract_download(file: str):
from fastapi.responses import Response
try:
safe=re.sub(r'[^A-Za-z0-9._-]', '_', file)
found=None
for base in ["/tmp/extracts", "/data/extracts", "/tmp", "/"]:
p=os.path.join(base, safe) if base!="/" else f"/{safe}"
try:
if os.path.isfile(p): found=p; break
except Exception:
continue
# fallback: glob any file starting with the target name
if not found:
import glob
for pat in [f"/tmp/extract_*_{safe}", f"/tmp/extract_*_{safe.replace('.txt','')}_*.txt"]:
m=glob.glob(pat)
if m: found=m[0]; break
if not found:
return JSONResponse({"error":"File not ready or expired"}, status_code=404)
with open(found,"rb") as fh:
data=fh.read()
return Response(content=data,
media_type="text/plain",
headers={"Content-Disposition":f'attachment; filename="{safe}"',
"Cache-Control":"no-store"})
except Exception as e:
traceback.print_exc()
return JSONResponse({"error":str(e)}, status_code=500)
@app.post("/fetch")
async def fetch_url(request: Request):
data=await request.json()
url=data.get("url","").strip()
if not url or not url.startswith("http"):
return JSONResponse({"status":"error","error":"Invalid URL"}, status_code=400)
job_id=uuid.uuid4().hex[:8]
jobs[job_id]={"status":"Queued...","pct":0}
import threading
threading.Thread(target=do_fetch, args=(job_id, url), daemon=True).start()
return {"job_id":job_id,"status":"Started"}
@app.post("/fetch/gofile")
async def fetch_gofile(request: Request):
data=await request.json()
url=data.get("url","").strip()
if not url or "gofile.io" not in url:
return JSONResponse({"status":"error","error":"Invalid GoFile URL"}, status_code=400)
job_id=uuid.uuid4().hex[:8]
jobs[job_id]={"status":"Queued GoFile...","pct":0}
import threading
threading.Thread(target=do_gofile, args=(job_id, url), daemon=True).start()
return {"job_id":job_id,"status":"Started"}
@app.post("/mediafire")
async def fetch_mediafire(request: Request):
data=await request.json()
url=data.get("url","").strip()
if not url or ("mediafire.com" not in url and "mediafireapp.com" not in url):
return JSONResponse({"status":"error","error":"Invalid MediaFire URL"}, status_code=400)
job_id=uuid.uuid4().hex[:8]
jobs[job_id]={"status":"Queued MediaFire...","pct":0}
import threading
threading.Thread(target=do_mediafire, args=(job_id, url), daemon=True).start()
return {"job_id":job_id,"status":"Started"}
@app.get("/fetch/progress")
async def fetch_progress(job_id: str):
return jobs.get(job_id, {"status":"Unknown job","pct":0})