Spaces:
Paused
Paused
Download app.py from kovinape/ulp-uploader: direct link, hf CLI and curl.
- Browser
- Download file 53.2 kB
-
https://huggingface.co/spaces/kovinape/ulp-uploader/resolve/main/app.py
- Command line
-
hf download hf://spaces/kovinape/ulp-uploader/app.py
-
curl -L -o app.py https://huggingface.co/spaces/kovinape/ulp-uploader/resolve/main/app.py
53.2 kB
| import os, uuid, re, shutil, traceback, time, subprocess, json | |
| from urllib.parse import unquote | |
| from fastapi import FastAPI, UploadFile, File, Request | |
| from fastapi.responses import HTMLResponse, JSONResponse | |
| from huggingface_hub import batch_bucket_files, hffs | |
| from concurrent.futures import ThreadPoolExecutor, as_completed | |
| BUCKET_ID = os.environ.get("HF_BUCKET_ID", "kovinape/ulp-bucket") | |
| SHARD_SIZE = 5 * 1024 * 1024 * 1024 | |
| app = FastAPI() | |
| jobs = {} | |
| HTML = """ | |
| <!DOCTYPE html> | |
| <html> | |
| <head><title>ULP Hub — Upload & Extract</title> | |
| <style> | |
| *{box-sizing:border-box;margin:0;padding:0} | |
| body{font-family:'Segoe UI',system-ui,sans-serif;min-height:100vh;color:#e5e9f0;background:radial-gradient(1200px 800px at 15% -10%,rgba(56,189,248,.14) 0%,transparent 60%),radial-gradient(1000px 700px at 110% 10%,rgba(56,189,248,.18) 0%,transparent 55%),linear-gradient(160deg,#0b0f17 0%,#141a26 50%,#0d1220 100%);background-attachment:fixed} | |
| body::before{content:'';position:fixed;inset:0;pointer-events:none;z-index:-1;background:linear-gradient(180deg,rgba(255,255,255,.04),transparent 30%),repeating-linear-gradient(0deg,transparent 0 2px,rgba(255,255,255,.014) 2px 3px),linear-gradient(110deg,transparent 42%,rgba(255,255,255,.06) 50%,transparent 58%)} | |
| h2,h3{color:#cbd5e1;font-weight:600;letter-spacing:.02em} | |
| code{background:rgba(255,255,255,.08);padding:2px 7px;border-radius:6px;font-size:.85em;color:#7dd3fc} | |
| nav{display:flex;gap:12px;padding:18px 22px;position:sticky;top:0;z-index:10;background:linear-gradient(180deg,rgba(15,20,30,.78),rgba(15,20,30,.55));backdrop-filter:blur(20px);-webkit-backdrop-filter:blur(20px);border-bottom:1px solid rgba(226,232,240,.12);box-shadow:0 6px 22px rgba(0,0,0,.45)} | |
| nav button{padding:11px 20px;border:1px solid rgba(226,232,240,.16);background:linear-gradient(165deg,rgba(203,213,225,.16),rgba(100,116,139,.05));color:#aab4c6;border-radius:10px;cursor:pointer;font-size:.92rem;backdrop-filter:blur(8px);transition:.25s;box-shadow:0 2px 10px rgba(0,0,0,.4),inset 0 1px 0 rgba(255,255,255,.2),inset 0 -2px 4px rgba(0,0,0,.25)} | |
| nav button:hover{color:#e2e8f0;border-color:rgba(125,211,252,.4);transform:translateY(-1px)} | |
| nav button.active{background:linear-gradient(135deg,rgba(56,189,248,.3),rgba(37,99,235,.22));color:#fff;border-color:rgba(125,211,252,.6);box-shadow:0 0 18px rgba(56,189,248,.28),inset 0 1px 0 rgba(255,255,255,.25)} | |
| .page{display:none;padding:24px;max-width:1000px;margin:0 auto} | |
| .page.active{display:block;animation:fadein .35s} | |
| @keyframes fadein{from{opacity:0;transform:translateY(6px)}to{opacity:1;transform:none}} | |
| .card{background:linear-gradient(165deg,rgba(148,163,184,.16),rgba(100,116,139,.08) 45%,rgba(51,65,85,.1));border:1px solid rgba(226,232,240,.14);border-radius:18px;padding:22px;margin:18px 0;backdrop-filter:blur(16px);-webkit-backdrop-filter:blur(16px);box-shadow:0 14px 34px rgba(0,0,0,.5),inset 0 1px 0 rgba(255,255,255,.14),inset 0 -1px 0 rgba(0,0,0,.35);position:relative} | |
| .card::before{content:'';position:absolute;inset:0;border-radius:18px;pointer-events:none;background:linear-gradient(115deg,transparent 30%,rgba(255,255,255,.09) 46%,rgba(255,255,255,.02) 52%,transparent 62%)} | |
| .card h3{margin-bottom:10px;font-size:1rem} | |
| .drop{border:2px dashed rgba(125,211,252,.5);border-radius:14px;padding:42px;text-align:center;cursor:pointer;background:rgba(56,189,248,.04);transition:.25s;color:#94a3b8} | |
| .drop:hover{border-color:#7dd3fc;background:rgba(56,189,248,.08)} | |
| .drop.dragover{border-color:#34d399;background:rgba(52,211,153,.12);color:#a7f3d0;box-shadow:0 0 24px rgba(52,211,153,.2)} | |
| .bar{width:100%;height:14px;background:rgba(255,255,255,.07);border-radius:10px;overflow:hidden;margin:12px 0;box-shadow:inset 0 2px 4px rgba(0,0,0,.4)} | |
| .bar>div{height:100%;width:0%;background:linear-gradient(90deg,#38bdf8,#6366f1,#a855f7);border-radius:10px;transition:width .3s;box-shadow:0 0 12px rgba(56,189,248,.5)} | |
| .bar.green>div{background:linear-gradient(90deg,#10b981,#34d399)} | |
| input[type=text],textarea{width:100%;padding:12px;border:1px solid rgba(255,255,255,.14);background:rgba(10,14,22,.7);color:#e2e8f0;border-radius:10px;font-family:ui-monospace,Menlo,monospace;font-size:.9rem;outline:none;transition:.2s} | |
| input[type=text]:focus,textarea:focus{border-color:#38bdf8;box-shadow:0 0 0 3px rgba(56,189,248,.18)} | |
| button.primary{background:linear-gradient(135deg,#38bdf8,#2563eb);color:#fff;border:0;padding:11px 22px;border-radius:10px;cursor:pointer;font-size:.92rem;font-weight:600;transition:.25s;box-shadow:0 4px 14px rgba(37,99,235,.4),inset 0 1px 0 rgba(255,255,255,.35),inset 0 -2px 4px rgba(0,0,0,.3)} | |
| button.primary:hover{transform:translateY(-1px);box-shadow:0 6px 20px rgba(56,189,248,.5),inset 0 1px 0 rgba(255,255,255,.45)} | |
| a.dl{display:inline-block;margin:6px 10px 6px 0;padding:10px 16px;background:linear-gradient(135deg,#10b981,#059669);color:#fff;border-radius:10px;text-decoration:none;font-size:.88rem;font-weight:600;transition:.2s;box-shadow:0 4px 14px rgba(16,185,129,.35),inset 0 1px 0 rgba(255,255,255,.3),inset 0 -2px 4px rgba(0,0,0,.25)} | |
| a.dl:hover{transform:translateY(-1px);box-shadow:0 6px 18px rgba(16,185,129,.5)} | |
| #s1,#fs,#gs,#ms,#extStatus,#extInfo{color:#94a3b8;font-size:.9rem} | |
| #extInfo{background:rgba(255,255,255,.05);border:1px solid rgba(255,255,255,.08);border-radius:10px;padding:10px 12px;font-family:ui-monospace,Menlo,monospace} | |
| </style> | |
| </head> | |
| <body> | |
| <nav> | |
| <button id="btn-upload" class="active" onclick="show('upload')">⚡ Upload ULP</button> | |
| <button id="btn-extract" onclick="show('extract')">🔍 Extract Combo</button> | |
| </nav> | |
| <div id="page-upload" class="page active"> | |
| <h2>⚡ Upload ULP → Private Bucket</h2> | |
| <div class="card"> | |
| <h3>From PC (direct, 5GB shards)</h3> | |
| <div class="drop" id="drop">Drop .txt here or click<br><small style="color:#64748b">100GB supported</small><input type="file" id="file" accept=".txt" style="display:none"></div> | |
| <div class="bar" id="p1" style="display:none"><div id="b1"></div></div> | |
| <div id="s1"></div> | |
| </div> | |
| <div class="card"> | |
| <h3>From GoFile</h3> | |
| <input type="text" id="gofile" placeholder="https://gofile.io/d/xxxx"> | |
| <button class="primary" onclick="fetchGofile()">Fetch GoFile → Bucket</button> | |
| <div id="gs" style="margin-top:8px"></div><div class="bar" id="p3" style="display:none"><div id="b3"></div></div> | |
| </div> | |
| <div class="card"> | |
| <h3>From MediaFire</h3> | |
| <input type="text" id="mediafire" placeholder="https://app.mediafire.com/folder/mdfd8hafxmmsy"> | |
| <button class="primary" onclick="fetchMediaFire()">Fetch MediaFire → Bucket</button> | |
| <div id="ms" style="margin-top:8px"></div><div class="bar" id="p4" style="display:none"><div id="b4"></div></div> | |
| </div> | |
| <div class="card"> | |
| <h3>From Direct Link</h3> | |
| <input type="text" id="url" placeholder="https://cdn2.filehost.org/download/xxxxx"> | |
| <button class="primary" onclick="fetchLink()">Fetch → Bucket</button> | |
| <div id="fs" style="margin-top:8px"></div><div class="bar" id="p2" style="display:none"><div id="b2"></div></div> | |
| </div> | |
| </div> | |
| <div id="page-extract" class="page"> | |
| <h2>🔍 Extract Combo from ULP</h2> | |
| <div class="card"> | |
| <p style="color:#94a3b8;margin-bottom:10px">Enter keywords — <b style="color:#cbd5e1">one per line</b> (e.g. <code>binomo.com</code>, <code>hotmail.com</code>)</p> | |
| <textarea id="kw" rows="6" placeholder="binomo hotmail.com Any keyword or site link"></textarea> | |
| <button class="primary" onclick="doExtract()" style="margin-top:12px">Extract</button> | |
| <div style="margin-top:12px;display:flex;align-items:center;gap:8px"> | |
| <button id="btn-adv" onclick="toggleAdv()" style="padding:8px 14px;border:1px solid rgba(226,232,240,.18);background:linear-gradient(165deg,rgba(203,213,225,.14),rgba(100,116,139,.04));color:#cbd5e1;border-radius:9px;cursor:pointer">⚙️ Advanced Settings</button> | |
| <span id="selCount" style="color:#64748b;font-size:.85rem"></span> | |
| </div> | |
| <div id="advPanel" style="display:none;margin-top:12px;border:1px solid rgba(226,232,240,.14);border-radius:12px;padding:12px;background:rgba(8,12,20,.5)"> | |
| <div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:8px"> | |
| <b style="color:#94a3b8;font-size:.9rem">Select ULPs to extract (newest first)</b> | |
| <div> | |
| <button class="mini" onclick="selAll()">Select all</button> | |
| <button class="mini" onclick="selNone()">Clear</button> | |
| <button class="mini" onclick="sel24()">Last 24h</button> | |
| </div> | |
| </div> | |
| <div id="ulplist" style="max-height:260px;overflow:auto;font-size:.85rem"></div> | |
| <p style="color:#64748b;font-size:.8rem;margin-top:8px">Leave everything unchecked → scan the whole bucket.</p> | |
| </div> | |
| <div id="extStatus" style="margin-top:12px"></div> | |
| <div class="bar" id="pE" style="display:none"><div id="bE"></div></div> | |
| <div id="extInfo" style="margin-top:10px; font-family:monospace"></div> | |
| <div id="extLinks" style="margin-top:14px"></div> | |
| </div> | |
| <style>.mini{padding:5px 10px;border:1px solid rgba(226,232,240,.16);background:rgba(148,163,184,.12);color:#cbd5e1;border-radius:7px;cursor:pointer;font-size:.78rem;margin-left:6px}.ulpitem{display:flex;align-items:center;gap:8px;padding:6px 8px;border-radius:7px;border:1px solid transparent;cursor:pointer}.ulpitem:hover{background:rgba(148,163,184,.1)}.ulpitem input{accent-color:#38bdf8}.ulpitem .nm{color:#cbd5e1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.ulpitem .sz{color:#64748b;font-size:.75rem;flex:none}</style> | |
| </div> | |
| <pre id="log"></pre> | |
| <script> | |
| function show(p){document.querySelectorAll('.page').forEach(e=>e.classList.remove('active'));document.querySelectorAll('nav button').forEach(e=>e.classList.remove('active'));document.getElementById('page-'+p).classList.add('active');document.getElementById('btn-'+p).classList.add('active');} | |
| const drop=document.getElementById('drop'), fileInput=document.getElementById('file'), b1=document.getElementById('b1'), p1=document.getElementById('p1'), s1=document.getElementById('s1'); | |
| drop.onclick=()=>fileInput.click(); | |
| drop.ondragover=e=>{e.preventDefault();drop.classList.add('dragover')}; | |
| drop.ondragleave=()=>drop.classList.remove('dragover'); | |
| drop.ondrop=e=>{e.preventDefault();drop.classList.remove('dragover');handle(e.dataTransfer.files[0])}; | |
| fileInput.onchange=e=>handle(e.target.files[0]); | |
| async function handle(file){ | |
| if(!file)return; | |
| p1.style.display='block'; s1.textContent=`Uploading ${file.name} (${(file.size/1024/1024/1024).toFixed(2)} GB)...`; | |
| const fd=new FormData(); fd.append('file', file); | |
| const xhr=new XMLHttpRequest(); | |
| xhr.upload.onprogress=e=>{ if(e.lengthComputable){ const pct=e.loaded/e.total*100; b1.style.width=pct+'%'; s1.textContent=`Uploading ${file.name} ${pct.toFixed(1)}%`; } }; | |
| xhr.onload=()=>{ s1.textContent=xhr.status==200?'✅ Done — in bucket':'❌ Failed'; b1.style.background='#10b981'; document.getElementById('log').textContent+=xhr.responseText+'\\n'; }; | |
| xhr.open('POST','/upload'); xhr.send(fd); | |
| } | |
| async function fetchLink(){ | |
| const url=document.getElementById('url').value.trim(); if(!url){alert('Paste URL');return;} | |
| document.getElementById('fs').textContent='Fetching...'; document.getElementById('p2').style.display='block'; | |
| const r=await fetch('/fetch',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({url})}); | |
| const j=await r.json(); if(j.job_id) poll(j.job_id,'fetch'); else document.getElementById('fs').textContent=JSON.stringify(j); | |
| } | |
| async function fetchGofile(){ | |
| const url=document.getElementById('gofile').value.trim(); if(!url){alert('Paste GoFile URL');return;} | |
| document.getElementById('gs').textContent='Resolving GoFile...'; document.getElementById('p3').style.display='block'; | |
| const r=await fetch('/fetch/gofile',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({url})}); | |
| const j=await r.json(); if(j.job_id) poll(j.job_id,'gofile'); else document.getElementById('gs').textContent=JSON.stringify(j); | |
| } | |
| async function poll(job_id,type){ | |
| const bar=document.getElementById(type=='gofile'?'b3':'b2'), st=document.getElementById(type=='gofile'?'gs':'fs'); | |
| const iv=setInterval(async()=>{ | |
| const r=await fetch('/fetch/progress?job_id='+job_id); const j=await r.json(); | |
| st.textContent=j.status+ (j.pct?` ${j.pct}%`:''); if(j.pct) bar.style.width=j.pct+'%'; | |
| if(j.status.includes('Done')||j.status.includes('Failed')) clearInterval(iv); | |
| },1000); | |
| } | |
| async function doExtract(){ | |
| const raw=document.getElementById('kw').value.trim(); if(!raw){alert('Enter keywords, one per line');return;} | |
| const targets=raw.split(/\\n/).map(s=>s.trim()).filter(Boolean); | |
| const files=Array.from(document.querySelectorAll('#ulplist input:checked')).map(c=>c.value); | |
| document.getElementById('extStatus').textContent='Starting extraction'+(files.length?` (${files.length} ULP${files.length>1?'s':''})`:' (whole bucket)')+ '...'; | |
| document.getElementById('pE').style.display='block'; | |
| document.getElementById('extLinks').innerHTML=''; | |
| const body={targets}; | |
| if(files.length) body.files=files; | |
| const r=await fetch('/extract',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(body)}); | |
| const j=await r.json(); if(!j.job_id){ document.getElementById('extStatus').textContent=JSON.stringify(j); return; } | |
| pollExtract(j.job_id); | |
| } | |
| let ulpCache=[]; | |
| async function toggleAdv(){ | |
| const p=document.getElementById('advPanel'); | |
| if(p.style.display==='none'){ | |
| p.style.display='block'; | |
| if(!ulpCache.length){ | |
| const ulp=document.getElementById('ulplist'); ulp.innerHTML='<span style="color:#64748b">Loading bucket files…</span>'; | |
| try{ | |
| const r=await fetch('/bucket/files'); const j=await r.json(); | |
| ulpCache=j.files||[]; | |
| }catch(e){ulpCache=[];} | |
| } | |
| renderUlp(); | |
| } else { p.style.display='none'; } | |
| } | |
| function renderUlp(){ | |
| const ulp=document.getElementById('ulplist'); | |
| if(!ulpCache.length){ ulp.innerHTML='<span style="color:#64748b">No ULPs in bucket.</span>'; updateSelCount(); return; } | |
| const fmt=s=>s>=1024**3?(s/1024**3).toFixed(1)+' GB':(s/1024**2|0)+' MB'; | |
| ulp.innerHTML=ulpCache.map(f=>`<label class="ulpitem" title="${f.name} (${fmt(f.size)}${f.uploaded_at?', '+f.uploaded_at.slice(0,16):''})" style="display:flex;align-items:center;gap:8px"> | |
| <input type="checkbox" value="${f.name.replace(/"/g,'"')}" data-ts="${f.ts||0}"> | |
| <span class="nm">${f.name}</span> | |
| <span class="sz">${fmt(f.size)}</span></label>`).join(''); | |
| ulp.onchange=()=>updateSelCount(); | |
| updateSelCount(); | |
| } | |
| function updateSelCount(){ | |
| const n=document.querySelectorAll('#ulplist input:checked').length; | |
| document.getElementById('selCount').textContent=n?`${n} selected`:'All ULPs (whole bucket)'; | |
| } | |
| function selAll(){ document.querySelectorAll('#ulplist input').forEach(c=>c.checked=true); updateSelCount(); } | |
| function selNone(){ document.querySelectorAll('#ulplist input').forEach(c=>c.checked=false); updateSelCount(); } | |
| function sel24(){ | |
| const cutoff=Date.now()/1000-86400; | |
| document.querySelectorAll('#ulplist input').forEach(c=>{ | |
| const ts=parseFloat(c.dataset.ts||'0'); | |
| c.checked=ts>0 && ts>=cutoff; | |
| }); | |
| updateSelCount(); | |
| } | |
| async function fetchMediaFire(){ | |
| const url=document.getElementById('mediafire').value.trim(); if(!url){alert('Paste MediaFire link');return;} | |
| document.getElementById('ms').textContent='Resolving MediaFire...'; document.getElementById('p4').style.display='block'; | |
| const r=await fetch('/mediafire',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({url})}); | |
| const j=await r.json(); if(j.job_id) pollMf(j.job_id); else document.getElementById('ms').textContent=JSON.stringify(j); | |
| } | |
| async function pollMf(job_id){ | |
| const bar=document.getElementById('b4'), st=document.getElementById('ms'); | |
| const iv=setInterval(async()=>{ | |
| const r=await fetch('/fetch/progress?job_id='+job_id); const j=await r.json(); | |
| st.textContent=j.status+(j.pct?` ${j.pct}%`:''); if(j.pct) bar.style.width=j.pct+'%'; | |
| if(j.status.includes('Done')||j.status.includes('Failed')) clearInterval(iv); | |
| },1000); | |
| } | |
| async function pollExtract(job_id){ | |
| const bar=document.getElementById('bE'), st=document.getElementById('extStatus'), info=document.getElementById('extInfo'), links=document.getElementById('extLinks'); | |
| const iv=setInterval(async()=>{ | |
| const r=await fetch('/extract/progress?job_id='+job_id); const j=await r.json(); | |
| st.textContent=j.status||''; if(j.pct) bar.style.width=j.pct+'%'; | |
| if(j.total_gb) info.textContent=`Scanned: ${j.scanned_gb||0}GB / ${j.total_gb}GB • ${j.scanned_lines||0} lines • Shards: ${j.shards} • Found: ${j.found||0} • ${j.speed||''}`; | |
| if(j.done){ | |
| clearInterval(iv); | |
| if(j.files && j.files.length){ | |
| links.innerHTML=j.files.map(f=>{ | |
| const href=f.download_url||`/extract/download?file=${encodeURIComponent(f.name)}`; | |
| return `<a class="dl" href="${href}" download="${f.name}">⬇ ${f.name} (${f.lines} lines)</a>`; | |
| }).join(''); | |
| } | |
| if(j.error) st.textContent+=' ❌ '+j.error; | |
| } | |
| },1000); | |
| } | |
| </script> | |
| </body> | |
| </html> | |
| """ | |
| async def index(): | |
| return HTML | |
| async def upload(file: UploadFile = File(...)): | |
| tmp_path = f"/tmp/{uuid.uuid4().hex}_{file.filename}" | |
| try: | |
| with open(tmp_path, "wb") as out: | |
| while True: | |
| chunk = await file.read(8*1024*1024) | |
| if not chunk: break | |
| out.write(chunk) | |
| size = os.path.getsize(tmp_path) | |
| shards=[] | |
| if size <= SHARD_SIZE: | |
| batch_bucket_files(BUCKET_ID, add=[(tmp_path, file.filename)]) | |
| shards=[file.filename] | |
| else: | |
| idx=0 | |
| with open(tmp_path, "rb") as fh: | |
| while True: | |
| chunk_path=f"{tmp_path}.part{idx:03d}" | |
| with open(chunk_path,"wb") as out: | |
| written=0 | |
| while written < SHARD_SIZE: | |
| data=fh.read(4*1024*1024) | |
| if not data: break | |
| out.write(data) | |
| written+=len(data) | |
| if written==0: | |
| try: os.remove(chunk_path) | |
| except: pass | |
| break | |
| if os.path.getsize(chunk_path)==0: break | |
| hf_name=f"{os.path.splitext(file.filename)[0]}_part{idx:03d}.txt" | |
| batch_bucket_files(BUCKET_ID, add=[(chunk_path, hf_name)]) | |
| shards.append(hf_name) | |
| try: os.remove(chunk_path) | |
| except: pass | |
| idx+=1 | |
| if fh.tell()>=size: break | |
| try: os.remove(tmp_path) | |
| except: pass | |
| return JSONResponse({"status":"ok","file":file.filename,"shards":shards,"size":size,"bucket":BUCKET_ID}) | |
| except Exception as e: | |
| traceback.print_exc() | |
| try: os.remove(tmp_path) | |
| except: pass | |
| return JSONResponse({"status":"error","error":str(e)}, status_code=500) | |
| def do_fetch(job_id, url): | |
| jobs[job_id]={"status":"Resolving...","pct":0} | |
| try: | |
| tmp_base=f"/tmp/fetch_{job_id}" | |
| os.makedirs(tmp_base, exist_ok=True) | |
| orig_url=url | |
| if "mediafire.com/file/" in url: | |
| try: | |
| import requests | |
| r=requests.get(url, timeout=30, headers={"User-Agent":"Mozilla/5.0"}) | |
| m=re.search(r'https://download\d+\.mediafire\.com/[^"]+', r.text) | |
| if m: | |
| url=m.group(0) | |
| jobs[job_id]={"status":"Resolved Mediafire direct link","pct":2} | |
| except Exception as e: | |
| jobs[job_id]={"status":f"Mediafire resolve failed: {e}","pct":0} | |
| fname=url.split("/")[-1].split("?")[0] or orig_url.split("/")[-1].split("?")[0] or "fetched.txt" | |
| fname=re.sub(r'[^A-Za-z0-9._ -]', '_', fname)[:80] or "fetched.txt" | |
| if not fname.endswith(".txt"): fname+=".txt" | |
| try: | |
| import requests | |
| h=requests.head(url, allow_redirects=True, timeout=10) | |
| cd=h.headers.get("content-disposition","") | |
| if "filename=" in cd: | |
| fname=re.sub(r'[^A-Za-z0-9._ -]', '_', cd.split("filename=")[-1].strip('\" ;'))[:80] | |
| if not fname.endswith(".txt"): fname+=".txt" | |
| except: pass | |
| tmp_path=os.path.join(tmp_base, fname) | |
| jobs[job_id]={"status":f"Downloading {fname}...","pct":5} | |
| try: | |
| cmd=["aria2c","-x","16","-s","16","-k","1M","-o",tmp_path,url,"--allow-overwrite=true","--auto-file-renaming=false","--console-log-level=warn"] | |
| proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) | |
| except FileNotFoundError: | |
| cmd=["curl","-L","-o",tmp_path,url] | |
| proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) | |
| while proc.poll() is None: | |
| try: | |
| cur=os.path.getsize(tmp_path) if os.path.exists(tmp_path) else 0 | |
| jobs[job_id]={"status":f"Downloading {fname} {cur//1024//1024}MB","pct":10,"speed":f"{cur//1024}KB"} | |
| except: pass | |
| time.sleep(1) | |
| if proc.returncode!=0 or not os.path.exists(tmp_path) or os.path.getsize(tmp_path)==0: | |
| jobs[job_id]={"status":f"Failed: {proc.returncode}","pct":0} | |
| return | |
| size=os.path.getsize(tmp_path) | |
| jobs[job_id]={"status":f"Downloaded {size//1024//1024}MB, uploading to bucket...","pct":50} | |
| if size <= SHARD_SIZE: | |
| batch_bucket_files(BUCKET_ID, add=[(tmp_path, fname)]) | |
| jobs[job_id]={"status":f"Done — {fname} ({size//1024//1024}MB) → bucket","pct":100} | |
| else: | |
| idx=0 | |
| shards=[] | |
| with open(tmp_path,"rb") as fh: | |
| while True: | |
| chunk_path=f"{tmp_path}.part{idx:03d}" | |
| with open(chunk_path,"wb") as out: | |
| written=0 | |
| while written < SHARD_SIZE: | |
| data=fh.read(4*1024*1024) | |
| if not data: break | |
| out.write(data) | |
| written+=len(data) | |
| if written==0: | |
| try: os.remove(chunk_path) | |
| except: pass | |
| break | |
| if not os.path.exists(chunk_path) or os.path.getsize(chunk_path)==0: break | |
| hf_name=f"{os.path.splitext(fname)[0]}_part{idx:03d}.txt" | |
| jobs[job_id]={"status":f"Uploading shard {idx+1} {written//1024//1024}MB...","pct":50+idx*10} | |
| batch_bucket_files(BUCKET_ID, add=[(chunk_path, hf_name)]) | |
| shards.append(hf_name) | |
| try: os.remove(chunk_path) | |
| except: pass | |
| idx+=1 | |
| if fh.tell()>=size: break | |
| jobs[job_id]={"status":f"Done — {len(shards)} shards of {fname} → bucket","pct":100} | |
| try: os.remove(tmp_path) | |
| except: pass | |
| try: shutil.rmtree(tmp_base) | |
| except: pass | |
| except Exception as e: | |
| traceback.print_exc() | |
| jobs[job_id]={"status":f"Failed: {e}","pct":0} | |
| def do_gofile(job_id, url): | |
| import hashlib, json as js | |
| jobs[job_id]={"status":"Resolving GoFile...","pct":2} | |
| try: | |
| m=re.search(r"gofile\.io/d/([A-Za-z0-9-]+)", url) | |
| if not m: | |
| jobs[job_id]={"status":"Invalid GoFile URL","pct":0}; return | |
| content_id=m.group(1) | |
| GOFILE_UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36" | |
| GOFILE_LANG="en-US" | |
| GOFILE_SALT="12af056dacea0b" | |
| def website_token(acc, offset=0): | |
| import hashlib as hl, time as tm | |
| window=int(tm.time()//14400)+offset | |
| raw=f"{GOFILE_UA}::{GOFILE_LANG}::{acc}::{window}::{GOFILE_SALT}" | |
| return hl.sha256(raw.encode()).hexdigest() | |
| out=subprocess.run(["curl","-sS","-X","POST","https://api.gofile.io/accounts","-H",f"User-Agent: {GOFILE_UA}","-H","Origin: https://gofile.io","-d",""], capture_output=True, text=True, timeout=30) | |
| token=(js.loads(out.stdout).get("data",{}) or {}).get("token") | |
| if not token: | |
| jobs[job_id]={"status":"No guest token","pct":0}; return | |
| jobs[job_id]={"status":"Guest token ok, resolving...","pct":10} | |
| def contents(acc, off, cid): | |
| cmd=["curl","-sS","-G",f"https://api.gofile.io/contents/{cid}","-H",f"Authorization: Bearer {acc}","-H",f"X-Website-Token: {website_token(acc,off)}","-H",f"X-BL: {GOFILE_LANG}","-H",f"User-Agent: {GOFILE_UA}","-H","Accept: */*","-H","Origin: https://gofile.io","-H","Referer: https://gofile.io/","--data-urlencode","contentFilter=","--data-urlencode","page=1","--data-urlencode","pageSize=1000","--data-urlencode","sortField=createTime","--data-urlencode","sortDirection=-1"] | |
| out=subprocess.run(cmd, capture_output=True, text=True, timeout=30) | |
| return js.loads(out.stdout) | |
| def collect(acc, off, cid): | |
| out=contents(acc, off, cid) | |
| st=out.get("status") | |
| if st=="error-notPremium" and off==0: | |
| out=contents(acc,-1,cid); st=out.get("status") | |
| if st!="ok": return None, f"gofile: {st}" | |
| d=out.get("data",{}) or {} | |
| if d.get("type")=="file" or d.get("link"): | |
| return [(d.get("name") or "gofile_file.txt", int(d.get("size") or 0), d.get("link") or "")], None | |
| entries=[] | |
| for child_id,c in (d.get("children") or {}).items(): | |
| if c.get("type")=="folder": | |
| sub,err=collect(acc,off,child_id) | |
| if err: return None, err | |
| entries.extend(sub) | |
| elif c.get("type")=="file": | |
| entries.append((c.get("name") or "gofile_file.txt", int(c.get("size") or 0), c.get("link") or "")) | |
| return entries, None | |
| files,err=None,None | |
| for off in (0,-1): | |
| files,err=collect(token,off,content_id) | |
| if files is not None and not err: break | |
| if not files: | |
| jobs[job_id]={"status":f"Failed: {err or 'no files'}","pct":0}; return | |
| files=[(n,s,u) for n,s,u in files if u] | |
| total=len(files) | |
| jobs[job_id]={"status":f"Found {total} file(s), downloading...","pct":15} | |
| for idx,(fname_raw,fsize_b,dl_url) in enumerate(files,1): | |
| fname=re.sub(r'[^A-Za-z0-9._ -]', '_', fname_raw)[:80] or f"gofile_{idx}.txt" | |
| tmp_base=f"/tmp/gofile_{job_id}_{idx}" | |
| os.makedirs(tmp_base, exist_ok=True) | |
| tmp_path=os.path.join(tmp_base, fname) | |
| cmd=["curl","-sS","-L","-o",tmp_path,dl_url,"-H",f"Cookie: accountToken={token}","-H",f"User-Agent: {GOFILE_UA}","-H","Referer: https://gofile.io/"] | |
| proc=subprocess.Popen(cmd) | |
| while proc.poll() is None: | |
| try: | |
| cur=os.path.getsize(tmp_path) if os.path.exists(tmp_path) else 0 | |
| pct=15+ int((idx-1)/total*70 + (cur/max(1,fsize_b))*70/total) if fsize_b else 15+int((idx-1)/total*70) | |
| jobs[job_id]={"status":f"Downloading {idx}/{total} {fname} {cur//1024//1024}MB/{fsize_b//1024//1024}MB","pct":pct} | |
| except: pass | |
| time.sleep(1) | |
| if proc.returncode!=0 or not os.path.exists(tmp_path) or os.path.getsize(tmp_path)==0: | |
| jobs[job_id]={"status":f"Failed {fname}","pct":0}; continue | |
| size=os.path.getsize(tmp_path) | |
| if size <= SHARD_SIZE: | |
| batch_bucket_files(BUCKET_ID, add=[(tmp_path, fname)]) | |
| else: | |
| with open(tmp_path,"rb") as fh: | |
| part=0 | |
| while True: | |
| chunk_path=f"{tmp_path}.part{part:03d}" | |
| with open(chunk_path,"wb") as out: | |
| written=0 | |
| while written < SHARD_SIZE: | |
| data=fh.read(4*1024*1024) | |
| if not data: break | |
| out.write(data) | |
| written+=len(data) | |
| if written==0: | |
| try: os.remove(chunk_path) | |
| except: pass | |
| break | |
| if not os.path.exists(chunk_path) or os.path.getsize(chunk_path)==0: break | |
| hf_name=f"{os.path.splitext(fname)[0]}_part{part:03d}.txt" | |
| batch_bucket_files(BUCKET_ID, add=[(chunk_path, hf_name)]) | |
| try: os.remove(chunk_path) | |
| except: pass | |
| part+=1 | |
| if fh.tell()>=size: break | |
| try: os.remove(tmp_path) | |
| except: pass | |
| try: shutil.rmtree(tmp_base) | |
| except: pass | |
| jobs[job_id]={"status":f"Done — {total} file(s) → bucket","pct":100} | |
| except Exception as e: | |
| traceback.print_exc() | |
| jobs[job_id]={"status":f"Failed: {e}","pct":0} | |
| # ---- MediaFire ---- | |
| MF_UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36" | |
| def _mf_list_folder(folder_key): | |
| files=[]; chunk=1 | |
| while True: | |
| u=(f"https://www.mediafire.com/api/1.4/folder/get_content.php?r=utga" | |
| f"&content_type=files&filter=all&order_by=name&order_direction=asc" | |
| f"&chunk={chunk}&version=1.5&folder_key={folder_key}&response_format=json") | |
| import requests | |
| r=requests.get(u, headers={"User-Agent":MF_UA}, timeout=30) | |
| fc=r.json().get("response",{}).get("folder_content",{}) | |
| for f in fc.get("files",[]): | |
| files.append({"quickkey":f.get("quickkey",""), | |
| "filename":f.get("filename","unknown.txt"), | |
| "size":int(f.get("size",0) or 0), | |
| "normal":(f.get("links",{}) or {}).get("normal_download","")}) | |
| if str(fc.get("more_chunks","no")).lower()!="yes" or chunk>200: break | |
| chunk+=1 | |
| return files | |
| def _mf_list_subfolders(folder_key): | |
| u=(f"https://www.mediafire.com/api/1.4/folder/get_content.php?r=utga" | |
| f"&content_type=folders&filter=all&chunk=1&version=1.5" | |
| f"&folder_key={folder_key}&response_format=json") | |
| import requests | |
| try: | |
| r=requests.get(u, headers={"User-Agent":MF_UA}, timeout=30) | |
| return r.json().get("response",{}).get("folder_content",{}).get("folders",[]) or [] | |
| except Exception: | |
| return [] | |
| def _mf_collect_folder(folder_key, depth=0): | |
| files=_mf_list_folder(folder_key) | |
| if depth<5: | |
| for sub in _mf_list_subfolders(folder_key): | |
| files+=_mf_collect_folder(sub.get("folderkey",""), depth+1) | |
| return files | |
| def _mf_resolve_file(quickkey, filename): | |
| import requests | |
| page=f"https://www.mediafire.com/file/{quickkey}/{filename}/file" | |
| try: | |
| r=requests.get(page, headers={"User-Agent":MF_UA}, timeout=30) | |
| for m in re.finditer(r'https://download\d+\.mediafire\.com/[^"\'\s<]+', r.text): | |
| return m.group(0).replace("&","&") | |
| except Exception: | |
| pass | |
| return None | |
| def _mf_dl_and_upload(job_id, dl_url, fname, base_status): | |
| import requests | |
| tmp_base=f"/tmp/mf_{job_id}" | |
| os.makedirs(tmp_base, exist_ok=True) | |
| tmp_path=os.path.join(tmp_base, fname) | |
| try: | |
| r=requests.get(dl_url, headers={"User-Agent":MF_UA}, stream=True, timeout=60, allow_redirects=True) | |
| if r.status_code!=200: | |
| jobs[job_id]["status"]=f"{base_status} → HTTP {r.status_code} {r.headers.get('content-type','')[:40]} len={r.headers.get('content-length','?')}" | |
| try: r.close() | |
| except: pass | |
| return False | |
| clen=int(r.headers.get("content-length",0) or 0) | |
| written=0 | |
| with open(tmp_path,"wb") as out: | |
| for chunk in r.iter_content(chunk_size=8*1024*1024): | |
| if not chunk: continue | |
| out.write(chunk); written+=len(chunk) | |
| if written%(64*1024*1024)==0: | |
| jobs[job_id]=dict(jobs.get(job_id,{})) | |
| jobs[job_id]["status"]=f"{base_status} {written//1024//1024}MB"+(f"/{clen//1024//1024}MB" if clen else "") | |
| try: r.close() | |
| except: pass | |
| if written==0: | |
| jobs[job_id]["status"]=f"{base_status} → empty body (0 bytes)" | |
| return False | |
| except Exception as e: | |
| jobs[job_id]["status"]=f"{base_status} → requests err: {str(e)[:160]}" | |
| try: shutil.rmtree(tmp_base) | |
| except: pass | |
| return False | |
| size=os.path.getsize(tmp_path) | |
| jobs[job_id]["status"]=f"{base_status} → uploading {size//1024//1024}MB to bucket..." | |
| if size<=SHARD_SIZE: | |
| batch_bucket_files(BUCKET_ID, add=[(tmp_path, fname)]) | |
| else: | |
| with open(tmp_path,"rb") as fh: | |
| part=0 | |
| while True: | |
| chunk_path=f"{tmp_path}.part{part:03d}" | |
| with open(chunk_path,"wb") as out: | |
| written=0 | |
| while written<SHARD_SIZE: | |
| data=fh.read(4*1024*1024) | |
| if not data: break | |
| out.write(data); written+=len(data) | |
| if written==0: | |
| try: os.remove(chunk_path) | |
| except: pass | |
| break | |
| if not os.path.exists(chunk_path) or os.path.getsize(chunk_path)==0: break | |
| hf_name=f"{os.path.splitext(fname)[0]}_part{part:03d}.txt" | |
| batch_bucket_files(BUCKET_ID, add=[(chunk_path, hf_name)]) | |
| try: os.remove(chunk_path) | |
| except: pass | |
| part+=1 | |
| if fh.tell()>=size: break | |
| try: os.remove(tmp_path) | |
| except: pass | |
| try: shutil.rmtree(tmp_base) | |
| except: pass | |
| return True | |
| def do_mediafire(job_id, url): | |
| import requests | |
| jobs[job_id]={"status":"Resolving MediaFire...","pct":2} | |
| try: | |
| folder_m=re.search(r'mediafire\.com/[^/]+/folder/([A-Za-z0-9]+)', url) | |
| if not folder_m: | |
| folder_m=re.search(r'mediafire\.com/folder/([A-Za-z0-9]+)', url) | |
| file_m=re.search(r'mediafire\.com/(?:file_premium|file)/([A-Za-z0-9]+)/([^/?&]+)', url) | |
| if folder_m and (not file_m or "mediafire.com/folder/" in url): | |
| folder_key=folder_m.group(1) | |
| jobs[job_id]={"status":"Listing folder files...","pct":4} | |
| all_files=_mf_collect_folder(folder_key) | |
| if not all_files: | |
| jobs[job_id]={"status":"No files found in folder","pct":0}; return | |
| total=len(all_files) | |
| total_gb=sum(f["size"] for f in all_files)/1024**3 | |
| jobs[job_id].update({"status":f"Found {total} files ({total_gb:.1f}GB), starting download...","pct":6,"total":total}) | |
| ok=0 | |
| for idx,f in enumerate(all_files,1): | |
| fname=re.sub(r'[^A-Za-z0-9._ -]', '_', f["filename"] or f"mf_{idx}.txt")[:80] or f"mf_{idx}.txt" | |
| if not fname.endswith(".txt"): fname+=".txt" | |
| dl_url=None | |
| if f.get("normal"): | |
| try: | |
| r=requests.get(f["normal"], headers={"User-Agent":MF_UA}, timeout=30, allow_redirects=True) | |
| m=re.search(r'https://download\d+\.mediafire\.com/[^"\'\s<]+', r.text) | |
| if m: dl_url=m.group(0).replace("&","&") | |
| except Exception: | |
| pass | |
| if not dl_url: | |
| dl_url=_mf_resolve_file(f["quickkey"], f["filename"]) | |
| if not dl_url: | |
| jobs[job_id].update({"status":f"Skip {idx}/{total}: can't resolve {fname}","pct":6+int((idx-1)/total*85)}) | |
| continue | |
| jobs[job_id].update({"status":f"Downloading {idx}/{total}: {fname}","pct":6+int((idx-1)/total*85)}) | |
| if _mf_dl_and_upload(job_id, dl_url, fname, f"Downloading {idx}/{total}"): | |
| ok+=1 | |
| jobs[job_id].update({"pct":6+int(idx/total*85)}) | |
| jobs[job_id]={"status":f"Done — {ok}/{total} file(s) → bucket","pct":100,"total":total} | |
| elif file_m: | |
| quickkey=file_m.group(1) | |
| fname=re.sub(r'[^A-Za-z0-9._ -]', '_', unquote(file_m.group(2))) or "mf_file.txt" | |
| if not fname.endswith(".txt"): fname+=".txt" | |
| dl_url=_mf_resolve_file(quickkey, file_m.group(2)) | |
| if not dl_url: | |
| try: | |
| r=requests.get(url, headers={"User-Agent":MF_UA}, timeout=30) | |
| m=re.search(r'https://download\d+\.mediafire\.com/[^"\'\s<]+', r.text) | |
| if m: dl_url=m.group(0).replace("&","&") | |
| except Exception: | |
| pass | |
| if not dl_url: | |
| jobs[job_id]={"status":"Failed: can't resolve MediaFire file","pct":0}; return | |
| jobs[job_id]={"status":f"Downloading {fname}...","pct":8} | |
| ok=_mf_dl_and_upload(job_id, dl_url, fname, f"Downloading {fname}") | |
| if ok: | |
| jobs[job_id]={"status":f"Done — {fname} → bucket","pct":100} | |
| else: | |
| jobs[job_id]={"status":"Invalid MediaFire URL (use /folder/ or /file/ link)","pct":0} | |
| except Exception as e: | |
| traceback.print_exc() | |
| jobs[job_id]={"status":f"Failed: {e}","pct":0} | |
| # ---- Extract ---- | |
| import hashlib | |
| def _hash_line(line: str) -> int: | |
| import hashlib as hl | |
| return int.from_bytes(hl.blake2b(line.encode("utf-8", "ignore"), digest_size=8).digest(), "big") | |
| def list_bucket_files(): | |
| from datetime import datetime, timezone | |
| infos = hffs.ls(f"buckets/{BUCKET_ID}", detail=True, refresh=True) | |
| out=[] | |
| for i in infos: | |
| if not isinstance(i, dict): continue | |
| nm=str(i.get("name","")) | |
| if not nm.lower().endswith(".txt"): continue | |
| base=nm.split(f"buckets/{BUCKET_ID}/")[-1] if f"buckets/{BUCKET_ID}/" in nm else nm | |
| uploaded_at=str(i.get("uploaded_at") or i.get("mtime") or "") | |
| ts=0 | |
| if uploaded_at: | |
| try: | |
| dt=datetime.fromisoformat(uploaded_at.replace("Z","+00:00")) | |
| ts=dt.timestamp() | |
| except Exception: | |
| try: | |
| dt=datetime.strptime(uploaded_at[:19], "%Y-%m-%d %H:%M:%S") | |
| ts=dt.replace(tzinfo=timezone.utc).timestamp() | |
| except Exception: | |
| ts=0 | |
| out.append({"name":base, | |
| "size":int(i.get("size",0) or 0), | |
| "uploaded_at":uploaded_at, | |
| "ts":ts}) | |
| out.sort(key=lambda x: x["uploaded_at"], reverse=True) | |
| return out | |
| def do_extract(job_id, targets, selected=None): | |
| jobs[job_id]={"status":"Listing bucket...","pct":5,"found":0,"shards":0,"total_gb":"0"} | |
| try: | |
| # list shards | |
| if selected: | |
| sel=set(selected) | |
| infos = hffs.ls(f"buckets/{BUCKET_ID}", detail=True, refresh=True) | |
| all_shards=[] | |
| for i in infos: | |
| if not isinstance(i, dict): continue | |
| nm=str(i.get("name","")) | |
| if not nm.lower().endswith(".txt"): continue | |
| base=nm.split(f"buckets/{BUCKET_ID}/")[-1] if f"buckets/{BUCKET_ID}/" in nm else nm | |
| if base in sel: | |
| all_shards.append(nm) | |
| else: | |
| paths = hffs.ls(f"buckets/{BUCKET_ID}", detail=False, refresh=True) | |
| all_shards = [p for p in paths if p.lower().endswith(".txt")] | |
| shards = all_shards | |
| per_gb=dict() | |
| try: | |
| for i in hffs.ls(f"buckets/{BUCKET_ID}", detail=True, refresh=True): | |
| if not isinstance(i, dict): continue | |
| nm=str(i.get("name","")) | |
| if not nm.lower().endswith(".txt"): continue | |
| base=nm.split(f"buckets/{BUCKET_ID}/")[-1] if f"buckets/{BUCKET_ID}/" in nm else nm | |
| per_gb[nm]=int(i.get("size",0) or 0)/1024**3 | |
| except Exception: | |
| pass | |
| total_gb = sum(per_gb.get(p,2.2) for p in shards) if shards else 0 | |
| jobs[job_id].update({"total_gb":f"{total_gb:.1f}","shards":len(shards)}) | |
| if not shards: | |
| jobs[job_id]={"status":"No txt in bucket (or none selected)","pct":0,"done":True}; return | |
| out_dir = f"/tmp/extract_{job_id}" | |
| os.makedirs(out_dir, exist_ok=True) | |
| needles = [t.lower() for t in targets] | |
| pat_path = os.path.join(out_dir, "patterns.txt") | |
| with open(pat_path,"w") as pf: | |
| for n in needles: | |
| pf.write(n+"\n") | |
| counts={t:0 for t in targets} | |
| seens={t:set() for t in targets} | |
| outs={t: open(os.path.join(out_dir, f"{t.replace('.','_')}.txt"),"w",encoding="utf-8",errors="ignore") for t in targets} | |
| total_scanned=0 | |
| done=0 | |
| def process_shard(hf_path): | |
| tmp_dl = os.path.join(out_dir, f"dl_{os.path.basename(hf_path)}") | |
| try: | |
| from huggingface_hub import download_bucket_files | |
| rel = hf_path.split(f"buckets/{BUCKET_ID}/")[-1] | |
| download_bucket_files(bucket_id=BUCKET_ID, files=[(rel, tmp_dl)]) | |
| local_path = tmp_dl | |
| except Exception: | |
| with hffs.open(hf_path,"r") as src, open(tmp_dl,"w",encoding="utf-8",errors="ignore") as dst: | |
| shutil.copyfileobj(src, dst) | |
| local_path = tmp_dl | |
| cmd=["grep","-a","-i","-F","-f",pat_path,"--",local_path] | |
| # use ripgrep if available | |
| try: | |
| import shutil as sh | |
| if sh.which("rg"): | |
| cmd=["rg","-a","-i","-F","-f",pat_path,"--",local_path] | |
| except: pass | |
| proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, errors="ignore", bufsize=1024*1024) | |
| local_counts={t:0 for t in targets} | |
| # accurate parse like bot | |
| import re as re2 | |
| _SCHEME_RE = re2.compile(r"^[a-zA-Z][a-zA-Z0-9+.-]*://") | |
| _DOMAIN_RE = re2.compile(r"(?:www\.)?([a-zA-Z0-9](?:[a-zA-Z0-9._-]*[a-zA-Z0-9])?\.[a-zA-Z]{2,})") | |
| def parse_line(l): | |
| l=l.strip() | |
| if not l: return None | |
| l=_SCHEME_RE.sub("", l) | |
| idx=-1 | |
| for d in "|;,\t:": | |
| i=l.find(d) | |
| if i!=-1 and (idx==-1 or i<idx): | |
| idx=i | |
| if idx==-1: | |
| m=_DOMAIN_RE.search(l) | |
| site=m.group(1).lower() if m else l.lower().rstrip("/") | |
| return site, [] | |
| site=l[:idx] | |
| m=_DOMAIN_RE.search(site) | |
| site=m.group(1).lower() if m else site.lower().rstrip("/") | |
| rest=l[idx+1:] | |
| for d in "|;,\t": | |
| rest=rest.replace(d,":") | |
| vals=[v for v in rest.split(":") if v] | |
| return site, vals | |
| def site_matches(site,tgt): | |
| t=[x for x in tgt.strip(".").split(".") if x] | |
| if not t: return False | |
| labels=site.split(".") | |
| if any(labels[i:i+len(t)]==t for i in range(len(labels)-len(t)+1)): | |
| return True | |
| h_labels=site.replace("-",".").split(".") | |
| h_labels=[x for x in h_labels if x] | |
| if any(h_labels[i:i+len(t)]==t for i in range(len(h_labels)-len(t)+1)): | |
| return True | |
| return False | |
| for line in proc.stdout: | |
| r=parse_line(line) | |
| if not r or not r[1]: continue | |
| site, vals = r | |
| joined=":".join(vals) | |
| for tgt, needle in zip(targets, needles): | |
| is_domain="." in tgt | |
| if is_domain: | |
| if not site_matches(site,tgt): continue | |
| else: | |
| if needle not in line.lower(): continue | |
| h=_hash_line(joined) | |
| if h not in seens[tgt]: | |
| seens[tgt].add(h) | |
| outs[tgt].write(joined+"\n") | |
| local_counts[tgt]+=1 | |
| proc.stdout.close(); proc.wait() | |
| try: os.remove(local_path) | |
| except: pass | |
| return local_counts | |
| workers = 16 if len(shards) > 16 else 8 | |
| done_set=set() | |
| with ThreadPoolExecutor(max_workers=workers) as ex: | |
| futs={ex.submit(process_shard, p): p for p in shards} | |
| for fut in as_completed(futs): | |
| p=futs[fut] | |
| done+=1 | |
| done_set.add(p) | |
| try: | |
| res=fut.result() | |
| for tgt in targets: | |
| counts[tgt]+=res.get(tgt,0) | |
| except Exception as e: | |
| traceback.print_exc() | |
| pct=10+ int(done/len(shards)*80) | |
| scanned_gb = sum(per_gb.get(dp,2.2) for dp in done_set) | |
| scanned_lines_est = done * 5000000 | |
| jobs[job_id].update({"status":f"Scanning {done}/{len(shards)}","pct":pct,"found":sum(counts.values()),"scanned_gb":f"{scanned_gb:.1f}","scanned_lines":f"{scanned_lines_est//1000000}M", "speed":"~48 MB/s"}) | |
| for fh in outs.values(): fh.close() | |
| files=[] | |
| for tgt in targets: | |
| path=os.path.join(out_dir, f"{tgt.replace('.','_')}.txt") | |
| if os.path.exists(path) and os.path.getsize(path)>0: | |
| name=f"{tgt}.txt" | |
| safe=re.sub(r'[^A-Za-z0-9._-]', '_', name) | |
| dl={} | |
| final=None | |
| # 1) persist on the Space's stable disk (survives restarts) + /tmp fallback | |
| try: | |
| pd="/data/extracts"; os.makedirs(pd, exist_ok=True) | |
| pd_path=os.path.join(pd, safe) | |
| shutil.copy(path, pd_path) | |
| final=pd_path | |
| dl["download_url"]=f"/extract/download?file={safe}" | |
| except Exception as e: | |
| print("persist fail:", e, flush=True) | |
| if final is None or (os.path.exists(final) and os.path.getsize(final)==0): | |
| try: | |
| td="/tmp/extracts"; os.makedirs(td, exist_ok=True) | |
| tp=os.path.join(td, safe) | |
| shutil.copy(path, tp); final=tp | |
| dl["download_url"]=f"/extract/download?file={safe}" | |
| except Exception as e: | |
| print("tmp fail:", e, flush=True) | |
| if final is None: | |
| final=f"/tmp/extract_{job_id}_{tgt.replace('.','_')}.txt" | |
| shutil.copy(path, final) | |
| # 2) push into the Space repo files/ folder — DISABLED: causes Space rebuild which wipes jobs | |
| # if you want repo backup, re-enable below (will trigger a restart ~30s later) | |
| """ | |
| try: | |
| from huggingface_hub import HfApi as _HfApi | |
| token=os.environ.get("HF_TOKEN") or os.environ.get("HF_TOKEN_WRITE") | |
| if token and os.environ.get("HF_SPACE_ID"): | |
| api=_HfApi(token=token) | |
| api.upload_file(path_or_fileobj=path, | |
| path_in_repo=f"files/extracts/{safe}", | |
| repo_id=os.environ["HF_SPACE_ID"], repo_type="space") | |
| dl["hf_url"]=(f"https://huggingface.co/spaces/{os.environ['HF_SPACE_ID']}/" | |
| f"resolve/main/files/extracts/{safe}?download=true") | |
| except Exception as e: | |
| print("repo push fail:", e, flush=True) | |
| """ | |
| files.append({"name":name,"lines":counts[tgt],"path":final, **dl}) | |
| else: | |
| try: os.remove(path) | |
| except: pass | |
| try: os.remove(pat_path) | |
| except: pass | |
| try: shutil.rmtree(out_dir) | |
| except: pass | |
| jobs[job_id].update({"status":f"Done — {sum(counts.values())} lines","pct":100,"done":True,"files":files,"found":sum(counts.values())}) | |
| except Exception as e: | |
| traceback.print_exc() | |
| jobs[job_id]={"status":f"Failed: {e}","pct":0,"done":True,"error":str(e)} | |
| async def extract(request: Request): | |
| data=await request.json() | |
| targets=data.get("targets",[]) | |
| if isinstance(targets, str): | |
| targets=[t.strip() for t in targets.split("\n") if t.strip()] | |
| targets=[t.strip() for t in targets if t.strip()] | |
| if not targets: | |
| return JSONResponse({"error":"Enter at least one keyword, one per line"}, status_code=400) | |
| if len(targets)>20: | |
| return JSONResponse({"error":"Max 20 keywords"}, status_code=400) | |
| selected=data.get("files") | |
| if isinstance(selected, str): | |
| selected=[s.strip() for s in selected.split("\n") if s.strip()] | |
| if isinstance(selected, list): | |
| selected=[s for s in selected if s and isinstance(s,str)] | |
| if not selected: selected=None | |
| job_id=uuid.uuid4().hex[:8] | |
| jobs[job_id]={"status":"Queued...","pct":0,"done":False} | |
| import threading | |
| threading.Thread(target=do_extract, args=(job_id, targets, selected), daemon=True).start() | |
| return {"job_id":job_id} | |
| async def bucket_files_list(): | |
| try: | |
| return {"files": list_bucket_files()} | |
| except Exception as e: | |
| return {"error":str(e), "files":[]} | |
| async def extract_progress(job_id: str): | |
| j=jobs.get(job_id, {"status":"Unknown","pct":0}) | |
| out={k:v for k,v in j.items() if not k.startswith("file_")} | |
| return out | |
| async def extract_download(file: str): | |
| from fastapi.responses import Response | |
| try: | |
| safe=re.sub(r'[^A-Za-z0-9._-]', '_', file) | |
| found=None | |
| for base in ["/tmp/extracts", "/data/extracts", "/tmp", "/"]: | |
| p=os.path.join(base, safe) if base!="/" else f"/{safe}" | |
| try: | |
| if os.path.isfile(p): found=p; break | |
| except Exception: | |
| continue | |
| # fallback: glob any file starting with the target name | |
| if not found: | |
| import glob | |
| for pat in [f"/tmp/extract_*_{safe}", f"/tmp/extract_*_{safe.replace('.txt','')}_*.txt"]: | |
| m=glob.glob(pat) | |
| if m: found=m[0]; break | |
| if not found: | |
| return JSONResponse({"error":"File not ready or expired"}, status_code=404) | |
| with open(found,"rb") as fh: | |
| data=fh.read() | |
| return Response(content=data, | |
| media_type="text/plain", | |
| headers={"Content-Disposition":f'attachment; filename="{safe}"', | |
| "Cache-Control":"no-store"}) | |
| except Exception as e: | |
| traceback.print_exc() | |
| return JSONResponse({"error":str(e)}, status_code=500) | |
| async def fetch_url(request: Request): | |
| data=await request.json() | |
| url=data.get("url","").strip() | |
| if not url or not url.startswith("http"): | |
| return JSONResponse({"status":"error","error":"Invalid URL"}, status_code=400) | |
| job_id=uuid.uuid4().hex[:8] | |
| jobs[job_id]={"status":"Queued...","pct":0} | |
| import threading | |
| threading.Thread(target=do_fetch, args=(job_id, url), daemon=True).start() | |
| return {"job_id":job_id,"status":"Started"} | |
| async def fetch_gofile(request: Request): | |
| data=await request.json() | |
| url=data.get("url","").strip() | |
| if not url or "gofile.io" not in url: | |
| return JSONResponse({"status":"error","error":"Invalid GoFile URL"}, status_code=400) | |
| job_id=uuid.uuid4().hex[:8] | |
| jobs[job_id]={"status":"Queued GoFile...","pct":0} | |
| import threading | |
| threading.Thread(target=do_gofile, args=(job_id, url), daemon=True).start() | |
| return {"job_id":job_id,"status":"Started"} | |
| async def fetch_mediafire(request: Request): | |
| data=await request.json() | |
| url=data.get("url","").strip() | |
| if not url or ("mediafire.com" not in url and "mediafireapp.com" not in url): | |
| return JSONResponse({"status":"error","error":"Invalid MediaFire URL"}, status_code=400) | |
| job_id=uuid.uuid4().hex[:8] | |
| jobs[job_id]={"status":"Queued MediaFire...","pct":0} | |
| import threading | |
| threading.Thread(target=do_mediafire, args=(job_id, url), daemon=True).start() | |
| return {"job_id":job_id,"status":"Started"} | |
| async def fetch_progress(job_id: str): | |
| return jobs.get(job_id, {"status":"Unknown job","pct":0}) | |