import os, uuid, re, shutil, traceback, time, subprocess, json from urllib.parse import unquote from fastapi import FastAPI, UploadFile, File, Request from fastapi.responses import HTMLResponse, JSONResponse from huggingface_hub import batch_bucket_files, hffs from concurrent.futures import ThreadPoolExecutor, as_completed BUCKET_ID = os.environ.get("HF_BUCKET_ID", "kovinape/ulp-bucket") SHARD_SIZE = 5 * 1024 * 1024 * 1024 app = FastAPI() jobs = {} HTML = """ ULP Hub β€” Upload & Extract

⚑ Upload ULP β†’ Private Bucket

From PC (direct, 5GB shards)

Drop .txt here or click
100GB supported

From GoFile

From MediaFire

From Direct Link

πŸ” Extract Combo from ULP

Enter keywords β€” one per line (e.g. binomo.com, hotmail.com)





"""

@app.get("/", response_class=HTMLResponse)
async def index():
    return HTML

@app.post("/upload")
async def upload(file: UploadFile = File(...)):
    tmp_path = f"/tmp/{uuid.uuid4().hex}_{file.filename}"
    try:
        with open(tmp_path, "wb") as out:
            while True:
                chunk = await file.read(8*1024*1024)
                if not chunk: break
                out.write(chunk)
        size = os.path.getsize(tmp_path)
        shards=[]
        if size <= SHARD_SIZE:
            batch_bucket_files(BUCKET_ID, add=[(tmp_path, file.filename)])
            shards=[file.filename]
        else:
            idx=0
            with open(tmp_path, "rb") as fh:
                while True:
                    chunk_path=f"{tmp_path}.part{idx:03d}"
                    with open(chunk_path,"wb") as out:
                        written=0
                        while written < SHARD_SIZE:
                            data=fh.read(4*1024*1024)
                            if not data: break
                            out.write(data)
                            written+=len(data)
                        if written==0:
                            try: os.remove(chunk_path)
                            except: pass
                            break
                    if os.path.getsize(chunk_path)==0: break
                    hf_name=f"{os.path.splitext(file.filename)[0]}_part{idx:03d}.txt"
                    batch_bucket_files(BUCKET_ID, add=[(chunk_path, hf_name)])
                    shards.append(hf_name)
                    try: os.remove(chunk_path)
                    except: pass
                    idx+=1
                    if fh.tell()>=size: break
        try: os.remove(tmp_path)
        except: pass
        return JSONResponse({"status":"ok","file":file.filename,"shards":shards,"size":size,"bucket":BUCKET_ID})
    except Exception as e:
        traceback.print_exc()
        try: os.remove(tmp_path)
        except: pass
        return JSONResponse({"status":"error","error":str(e)}, status_code=500)

def do_fetch(job_id, url):
    jobs[job_id]={"status":"Resolving...","pct":0}
    try:
        tmp_base=f"/tmp/fetch_{job_id}"
        os.makedirs(tmp_base, exist_ok=True)
        orig_url=url
        if "mediafire.com/file/" in url:
            try:
                import requests
                r=requests.get(url, timeout=30, headers={"User-Agent":"Mozilla/5.0"})
                m=re.search(r'https://download\d+\.mediafire\.com/[^"]+', r.text)
                if m:
                    url=m.group(0)
                    jobs[job_id]={"status":"Resolved Mediafire direct link","pct":2}
            except Exception as e:
                jobs[job_id]={"status":f"Mediafire resolve failed: {e}","pct":0}
        fname=url.split("/")[-1].split("?")[0] or orig_url.split("/")[-1].split("?")[0] or "fetched.txt"
        fname=re.sub(r'[^A-Za-z0-9._ -]', '_', fname)[:80] or "fetched.txt"
        if not fname.endswith(".txt"): fname+=".txt"
        try:
            import requests
            h=requests.head(url, allow_redirects=True, timeout=10)
            cd=h.headers.get("content-disposition","")
            if "filename=" in cd:
                fname=re.sub(r'[^A-Za-z0-9._ -]', '_', cd.split("filename=")[-1].strip('\" ;'))[:80]
                if not fname.endswith(".txt"): fname+=".txt"
        except: pass
        tmp_path=os.path.join(tmp_base, fname)
        jobs[job_id]={"status":f"Downloading {fname}...","pct":5}
        try:
            cmd=["aria2c","-x","16","-s","16","-k","1M","-o",tmp_path,url,"--allow-overwrite=true","--auto-file-renaming=false","--console-log-level=warn"]
            proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        except FileNotFoundError:
            cmd=["curl","-L","-o",tmp_path,url]
            proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        while proc.poll() is None:
            try:
                cur=os.path.getsize(tmp_path) if os.path.exists(tmp_path) else 0
                jobs[job_id]={"status":f"Downloading {fname} {cur//1024//1024}MB","pct":10,"speed":f"{cur//1024}KB"}
            except: pass
            time.sleep(1)
        if proc.returncode!=0 or not os.path.exists(tmp_path) or os.path.getsize(tmp_path)==0:
            jobs[job_id]={"status":f"Failed: {proc.returncode}","pct":0}
            return
        size=os.path.getsize(tmp_path)
        jobs[job_id]={"status":f"Downloaded {size//1024//1024}MB, uploading to bucket...","pct":50}
        if size <= SHARD_SIZE:
            batch_bucket_files(BUCKET_ID, add=[(tmp_path, fname)])
            jobs[job_id]={"status":f"Done β€” {fname} ({size//1024//1024}MB) β†’ bucket","pct":100}
        else:
            idx=0
            shards=[]
            with open(tmp_path,"rb") as fh:
                while True:
                    chunk_path=f"{tmp_path}.part{idx:03d}"
                    with open(chunk_path,"wb") as out:
                        written=0
                        while written < SHARD_SIZE:
                            data=fh.read(4*1024*1024)
                            if not data: break
                            out.write(data)
                            written+=len(data)
                        if written==0:
                            try: os.remove(chunk_path)
                            except: pass
                            break
                    if not os.path.exists(chunk_path) or os.path.getsize(chunk_path)==0: break
                    hf_name=f"{os.path.splitext(fname)[0]}_part{idx:03d}.txt"
                    jobs[job_id]={"status":f"Uploading shard {idx+1} {written//1024//1024}MB...","pct":50+idx*10}
                    batch_bucket_files(BUCKET_ID, add=[(chunk_path, hf_name)])
                    shards.append(hf_name)
                    try: os.remove(chunk_path)
                    except: pass
                    idx+=1
                    if fh.tell()>=size: break
            jobs[job_id]={"status":f"Done β€” {len(shards)} shards of {fname} β†’ bucket","pct":100}
        try: os.remove(tmp_path)
        except: pass
        try: shutil.rmtree(tmp_base)
        except: pass
    except Exception as e:
        traceback.print_exc()
        jobs[job_id]={"status":f"Failed: {e}","pct":0}

def do_gofile(job_id, url):
    import hashlib, json as js
    jobs[job_id]={"status":"Resolving GoFile...","pct":2}
    try:
        m=re.search(r"gofile\.io/d/([A-Za-z0-9-]+)", url)
        if not m:
            jobs[job_id]={"status":"Invalid GoFile URL","pct":0}; return
        content_id=m.group(1)
        GOFILE_UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
        GOFILE_LANG="en-US"
        GOFILE_SALT="12af056dacea0b"
        def website_token(acc, offset=0):
            import hashlib as hl, time as tm
            window=int(tm.time()//14400)+offset
            raw=f"{GOFILE_UA}::{GOFILE_LANG}::{acc}::{window}::{GOFILE_SALT}"
            return hl.sha256(raw.encode()).hexdigest()
        out=subprocess.run(["curl","-sS","-X","POST","https://api.gofile.io/accounts","-H",f"User-Agent: {GOFILE_UA}","-H","Origin: https://gofile.io","-d",""], capture_output=True, text=True, timeout=30)
        token=(js.loads(out.stdout).get("data",{}) or {}).get("token")
        if not token:
            jobs[job_id]={"status":"No guest token","pct":0}; return
        jobs[job_id]={"status":"Guest token ok, resolving...","pct":10}
        def contents(acc, off, cid):
            cmd=["curl","-sS","-G",f"https://api.gofile.io/contents/{cid}","-H",f"Authorization: Bearer {acc}","-H",f"X-Website-Token: {website_token(acc,off)}","-H",f"X-BL: {GOFILE_LANG}","-H",f"User-Agent: {GOFILE_UA}","-H","Accept: */*","-H","Origin: https://gofile.io","-H","Referer: https://gofile.io/","--data-urlencode","contentFilter=","--data-urlencode","page=1","--data-urlencode","pageSize=1000","--data-urlencode","sortField=createTime","--data-urlencode","sortDirection=-1"]
            out=subprocess.run(cmd, capture_output=True, text=True, timeout=30)
            return js.loads(out.stdout)
        def collect(acc, off, cid):
            out=contents(acc, off, cid)
            st=out.get("status")
            if st=="error-notPremium" and off==0:
                out=contents(acc,-1,cid); st=out.get("status")
            if st!="ok": return None, f"gofile: {st}"
            d=out.get("data",{}) or {}
            if d.get("type")=="file" or d.get("link"):
                return [(d.get("name") or "gofile_file.txt", int(d.get("size") or 0), d.get("link") or "")], None
            entries=[]
            for child_id,c in (d.get("children") or {}).items():
                if c.get("type")=="folder":
                    sub,err=collect(acc,off,child_id)
                    if err: return None, err
                    entries.extend(sub)
                elif c.get("type")=="file":
                    entries.append((c.get("name") or "gofile_file.txt", int(c.get("size") or 0), c.get("link") or ""))
            return entries, None
        files,err=None,None
        for off in (0,-1):
            files,err=collect(token,off,content_id)
            if files is not None and not err: break
        if not files:
            jobs[job_id]={"status":f"Failed: {err or 'no files'}","pct":0}; return
        files=[(n,s,u) for n,s,u in files if u]
        total=len(files)
        jobs[job_id]={"status":f"Found {total} file(s), downloading...","pct":15}
        for idx,(fname_raw,fsize_b,dl_url) in enumerate(files,1):
            fname=re.sub(r'[^A-Za-z0-9._ -]', '_', fname_raw)[:80] or f"gofile_{idx}.txt"
            tmp_base=f"/tmp/gofile_{job_id}_{idx}"
            os.makedirs(tmp_base, exist_ok=True)
            tmp_path=os.path.join(tmp_base, fname)
            cmd=["curl","-sS","-L","-o",tmp_path,dl_url,"-H",f"Cookie: accountToken={token}","-H",f"User-Agent: {GOFILE_UA}","-H","Referer: https://gofile.io/"]
            proc=subprocess.Popen(cmd)
            while proc.poll() is None:
                try:
                    cur=os.path.getsize(tmp_path) if os.path.exists(tmp_path) else 0
                    pct=15+ int((idx-1)/total*70 + (cur/max(1,fsize_b))*70/total) if fsize_b else 15+int((idx-1)/total*70)
                    jobs[job_id]={"status":f"Downloading {idx}/{total} {fname} {cur//1024//1024}MB/{fsize_b//1024//1024}MB","pct":pct}
                except: pass
                time.sleep(1)
            if proc.returncode!=0 or not os.path.exists(tmp_path) or os.path.getsize(tmp_path)==0:
                jobs[job_id]={"status":f"Failed {fname}","pct":0}; continue
            size=os.path.getsize(tmp_path)
            if size <= SHARD_SIZE:
                batch_bucket_files(BUCKET_ID, add=[(tmp_path, fname)])
            else:
                with open(tmp_path,"rb") as fh:
                    part=0
                    while True:
                        chunk_path=f"{tmp_path}.part{part:03d}"
                        with open(chunk_path,"wb") as out:
                            written=0
                            while written < SHARD_SIZE:
                                data=fh.read(4*1024*1024)
                                if not data: break
                                out.write(data)
                                written+=len(data)
                            if written==0:
                                try: os.remove(chunk_path)
                                except: pass
                                break
                        if not os.path.exists(chunk_path) or os.path.getsize(chunk_path)==0: break
                        hf_name=f"{os.path.splitext(fname)[0]}_part{part:03d}.txt"
                        batch_bucket_files(BUCKET_ID, add=[(chunk_path, hf_name)])
                        try: os.remove(chunk_path)
                        except: pass
                        part+=1
                        if fh.tell()>=size: break
            try: os.remove(tmp_path)
            except: pass
            try: shutil.rmtree(tmp_base)
            except: pass
        jobs[job_id]={"status":f"Done β€” {total} file(s) β†’ bucket","pct":100}
    except Exception as e:
        traceback.print_exc()
        jobs[job_id]={"status":f"Failed: {e}","pct":0}

# ---- MediaFire ----
MF_UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"

def _mf_list_folder(folder_key):
    files=[]; chunk=1
    while True:
        u=(f"https://www.mediafire.com/api/1.4/folder/get_content.php?r=utga"
           f"&content_type=files&filter=all&order_by=name&order_direction=asc"
           f"&chunk={chunk}&version=1.5&folder_key={folder_key}&response_format=json")
        import requests
        r=requests.get(u, headers={"User-Agent":MF_UA}, timeout=30)
        fc=r.json().get("response",{}).get("folder_content",{})
        for f in fc.get("files",[]):
            files.append({"quickkey":f.get("quickkey",""),
                          "filename":f.get("filename","unknown.txt"),
                          "size":int(f.get("size",0) or 0),
                          "normal":(f.get("links",{}) or {}).get("normal_download","")})
        if str(fc.get("more_chunks","no")).lower()!="yes" or chunk>200: break
        chunk+=1
    return files

def _mf_list_subfolders(folder_key):
    u=(f"https://www.mediafire.com/api/1.4/folder/get_content.php?r=utga"
       f"&content_type=folders&filter=all&chunk=1&version=1.5"
       f"&folder_key={folder_key}&response_format=json")
    import requests
    try:
        r=requests.get(u, headers={"User-Agent":MF_UA}, timeout=30)
        return r.json().get("response",{}).get("folder_content",{}).get("folders",[]) or []
    except Exception:
        return []

def _mf_collect_folder(folder_key, depth=0):
    files=_mf_list_folder(folder_key)
    if depth<5:
        for sub in _mf_list_subfolders(folder_key):
            files+=_mf_collect_folder(sub.get("folderkey",""), depth+1)
    return files

def _mf_resolve_file(quickkey, filename):
    import requests
    page=f"https://www.mediafire.com/file/{quickkey}/{filename}/file"
    try:
        r=requests.get(page, headers={"User-Agent":MF_UA}, timeout=30)
        for m in re.finditer(r'https://download\d+\.mediafire\.com/[^"\'\s<]+', r.text):
            return m.group(0).replace("&","&")
    except Exception:
        pass
    return None

def _mf_dl_and_upload(job_id, dl_url, fname, base_status):
    import requests
    tmp_base=f"/tmp/mf_{job_id}"
    os.makedirs(tmp_base, exist_ok=True)
    tmp_path=os.path.join(tmp_base, fname)
    try:
        r=requests.get(dl_url, headers={"User-Agent":MF_UA}, stream=True, timeout=60, allow_redirects=True)
        if r.status_code!=200:
            jobs[job_id]["status"]=f"{base_status} β†’ HTTP {r.status_code} {r.headers.get('content-type','')[:40]} len={r.headers.get('content-length','?')}"
            try: r.close()
            except: pass
            return False
        clen=int(r.headers.get("content-length",0) or 0)
        written=0
        with open(tmp_path,"wb") as out:
            for chunk in r.iter_content(chunk_size=8*1024*1024):
                if not chunk: continue
                out.write(chunk); written+=len(chunk)
                if written%(64*1024*1024)==0:
                    jobs[job_id]=dict(jobs.get(job_id,{}))
                    jobs[job_id]["status"]=f"{base_status} {written//1024//1024}MB"+(f"/{clen//1024//1024}MB" if clen else "")
        try: r.close()
        except: pass
        if written==0:
            jobs[job_id]["status"]=f"{base_status} β†’ empty body (0 bytes)"
            return False
    except Exception as e:
        jobs[job_id]["status"]=f"{base_status} β†’ requests err: {str(e)[:160]}"
        try: shutil.rmtree(tmp_base)
        except: pass
        return False
    size=os.path.getsize(tmp_path)
    jobs[job_id]["status"]=f"{base_status} β†’ uploading {size//1024//1024}MB to bucket..."
    if size<=SHARD_SIZE:
        batch_bucket_files(BUCKET_ID, add=[(tmp_path, fname)])
    else:
        with open(tmp_path,"rb") as fh:
            part=0
            while True:
                chunk_path=f"{tmp_path}.part{part:03d}"
                with open(chunk_path,"wb") as out:
                    written=0
                    while written=size: break
    try: os.remove(tmp_path)
    except: pass
    try: shutil.rmtree(tmp_base)
    except: pass
    return True

def do_mediafire(job_id, url):
    import requests
    jobs[job_id]={"status":"Resolving MediaFire...","pct":2}
    try:
        folder_m=re.search(r'mediafire\.com/[^/]+/folder/([A-Za-z0-9]+)', url)
        if not folder_m:
            folder_m=re.search(r'mediafire\.com/folder/([A-Za-z0-9]+)', url)
        file_m=re.search(r'mediafire\.com/(?:file_premium|file)/([A-Za-z0-9]+)/([^/?&]+)', url)

        if folder_m and (not file_m or "mediafire.com/folder/" in url):
            folder_key=folder_m.group(1)
            jobs[job_id]={"status":"Listing folder files...","pct":4}
            all_files=_mf_collect_folder(folder_key)
            if not all_files:
                jobs[job_id]={"status":"No files found in folder","pct":0}; return
            total=len(all_files)
            total_gb=sum(f["size"] for f in all_files)/1024**3
            jobs[job_id].update({"status":f"Found {total} files ({total_gb:.1f}GB), starting download...","pct":6,"total":total})
            ok=0
            for idx,f in enumerate(all_files,1):
                fname=re.sub(r'[^A-Za-z0-9._ -]', '_', f["filename"] or f"mf_{idx}.txt")[:80] or f"mf_{idx}.txt"
                if not fname.endswith(".txt"): fname+=".txt"
                dl_url=None
                if f.get("normal"):
                    try:
                        r=requests.get(f["normal"], headers={"User-Agent":MF_UA}, timeout=30, allow_redirects=True)
                        m=re.search(r'https://download\d+\.mediafire\.com/[^"\'\s<]+', r.text)
                        if m: dl_url=m.group(0).replace("&","&")
                    except Exception:
                        pass
                if not dl_url:
                    dl_url=_mf_resolve_file(f["quickkey"], f["filename"])
                if not dl_url:
                    jobs[job_id].update({"status":f"Skip {idx}/{total}: can't resolve {fname}","pct":6+int((idx-1)/total*85)})
                    continue
                jobs[job_id].update({"status":f"Downloading {idx}/{total}: {fname}","pct":6+int((idx-1)/total*85)})
                if _mf_dl_and_upload(job_id, dl_url, fname, f"Downloading {idx}/{total}"):
                    ok+=1
                jobs[job_id].update({"pct":6+int(idx/total*85)})
            jobs[job_id]={"status":f"Done β€” {ok}/{total} file(s) β†’ bucket","pct":100,"total":total}
        elif file_m:
            quickkey=file_m.group(1)
            fname=re.sub(r'[^A-Za-z0-9._ -]', '_', unquote(file_m.group(2))) or "mf_file.txt"
            if not fname.endswith(".txt"): fname+=".txt"
            dl_url=_mf_resolve_file(quickkey, file_m.group(2))
            if not dl_url:
                try:
                    r=requests.get(url, headers={"User-Agent":MF_UA}, timeout=30)
                    m=re.search(r'https://download\d+\.mediafire\.com/[^"\'\s<]+', r.text)
                    if m: dl_url=m.group(0).replace("&","&")
                except Exception:
                    pass
            if not dl_url:
                jobs[job_id]={"status":"Failed: can't resolve MediaFire file","pct":0}; return
            jobs[job_id]={"status":f"Downloading {fname}...","pct":8}
            ok=_mf_dl_and_upload(job_id, dl_url, fname, f"Downloading {fname}")
            if ok:
                jobs[job_id]={"status":f"Done β€” {fname} β†’ bucket","pct":100}
        else:
            jobs[job_id]={"status":"Invalid MediaFire URL (use /folder/ or /file/ link)","pct":0}
    except Exception as e:
        traceback.print_exc()
        jobs[job_id]={"status":f"Failed: {e}","pct":0}

# ---- Extract ----
import hashlib

def _hash_line(line: str) -> int:
    import hashlib as hl
    return int.from_bytes(hl.blake2b(line.encode("utf-8", "ignore"), digest_size=8).digest(), "big")

def list_bucket_files():
    from datetime import datetime, timezone
    infos = hffs.ls(f"buckets/{BUCKET_ID}", detail=True, refresh=True)
    out=[]
    for i in infos:
        if not isinstance(i, dict): continue
        nm=str(i.get("name",""))
        if not nm.lower().endswith(".txt"): continue
        base=nm.split(f"buckets/{BUCKET_ID}/")[-1] if f"buckets/{BUCKET_ID}/" in nm else nm
        uploaded_at=str(i.get("uploaded_at") or i.get("mtime") or "")
        ts=0
        if uploaded_at:
            try:
                dt=datetime.fromisoformat(uploaded_at.replace("Z","+00:00"))
                ts=dt.timestamp()
            except Exception:
                try:
                    dt=datetime.strptime(uploaded_at[:19], "%Y-%m-%d %H:%M:%S")
                    ts=dt.replace(tzinfo=timezone.utc).timestamp()
                except Exception:
                    ts=0
        out.append({"name":base,
                    "size":int(i.get("size",0) or 0),
                    "uploaded_at":uploaded_at,
                    "ts":ts})
    out.sort(key=lambda x: x["uploaded_at"], reverse=True)
    return out

def do_extract(job_id, targets, selected=None):
    jobs[job_id]={"status":"Listing bucket...","pct":5,"found":0,"shards":0,"total_gb":"0"}
    try:
        # list shards
        if selected:
            sel=set(selected)
            infos = hffs.ls(f"buckets/{BUCKET_ID}", detail=True, refresh=True)
            all_shards=[]
            for i in infos:
                if not isinstance(i, dict): continue
                nm=str(i.get("name",""))
                if not nm.lower().endswith(".txt"): continue
                base=nm.split(f"buckets/{BUCKET_ID}/")[-1] if f"buckets/{BUCKET_ID}/" in nm else nm
                if base in sel:
                    all_shards.append(nm)
        else:
            paths = hffs.ls(f"buckets/{BUCKET_ID}", detail=False, refresh=True)
            all_shards = [p for p in paths if p.lower().endswith(".txt")]
        shards = all_shards
        per_gb=dict()
        try:
            for i in hffs.ls(f"buckets/{BUCKET_ID}", detail=True, refresh=True):
                if not isinstance(i, dict): continue
                nm=str(i.get("name",""))
                if not nm.lower().endswith(".txt"): continue
                base=nm.split(f"buckets/{BUCKET_ID}/")[-1] if f"buckets/{BUCKET_ID}/" in nm else nm
                per_gb[nm]=int(i.get("size",0) or 0)/1024**3
        except Exception:
            pass
        total_gb = sum(per_gb.get(p,2.2) for p in shards) if shards else 0
        jobs[job_id].update({"total_gb":f"{total_gb:.1f}","shards":len(shards)})
        if not shards:
            jobs[job_id]={"status":"No txt in bucket (or none selected)","pct":0,"done":True}; return
        out_dir = f"/tmp/extract_{job_id}"
        os.makedirs(out_dir, exist_ok=True)
        needles = [t.lower() for t in targets]
        pat_path = os.path.join(out_dir, "patterns.txt")
        with open(pat_path,"w") as pf:
            for n in needles:
                pf.write(n+"\n")
        counts={t:0 for t in targets}
        seens={t:set() for t in targets}
        outs={t: open(os.path.join(out_dir, f"{t.replace('.','_')}.txt"),"w",encoding="utf-8",errors="ignore") for t in targets}
        total_scanned=0
        done=0
        def process_shard(hf_path):
            tmp_dl = os.path.join(out_dir, f"dl_{os.path.basename(hf_path)}")
            try:
                from huggingface_hub import download_bucket_files
                rel = hf_path.split(f"buckets/{BUCKET_ID}/")[-1]
                download_bucket_files(bucket_id=BUCKET_ID, files=[(rel, tmp_dl)])
                local_path = tmp_dl
            except Exception:
                with hffs.open(hf_path,"r") as src, open(tmp_dl,"w",encoding="utf-8",errors="ignore") as dst:
                    shutil.copyfileobj(src, dst)
                local_path = tmp_dl
            cmd=["grep","-a","-i","-F","-f",pat_path,"--",local_path]
            # use ripgrep if available
            try:
                import shutil as sh
                if sh.which("rg"):
                    cmd=["rg","-a","-i","-F","-f",pat_path,"--",local_path]
            except: pass
            proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, errors="ignore", bufsize=1024*1024)
            local_counts={t:0 for t in targets}
            # accurate parse like bot
            import re as re2
            _SCHEME_RE = re2.compile(r"^[a-zA-Z][a-zA-Z0-9+.-]*://")
            _DOMAIN_RE = re2.compile(r"(?:www\.)?([a-zA-Z0-9](?:[a-zA-Z0-9._-]*[a-zA-Z0-9])?\.[a-zA-Z]{2,})")
            def parse_line(l):
                l=l.strip()
                if not l: return None
                l=_SCHEME_RE.sub("", l)
                idx=-1
                for d in "|;,\t:":
                    i=l.find(d)
                    if i!=-1 and (idx==-1 or i 16 else 8
        done_set=set()
        with ThreadPoolExecutor(max_workers=workers) as ex:
            futs={ex.submit(process_shard, p): p for p in shards}
            for fut in as_completed(futs):
                p=futs[fut]
                done+=1
                done_set.add(p)
                try:
                    res=fut.result()
                    for tgt in targets:
                        counts[tgt]+=res.get(tgt,0)
                except Exception as e:
                    traceback.print_exc()
                pct=10+ int(done/len(shards)*80)
                scanned_gb = sum(per_gb.get(dp,2.2) for dp in done_set)
                scanned_lines_est = done * 5000000
                jobs[job_id].update({"status":f"Scanning {done}/{len(shards)}","pct":pct,"found":sum(counts.values()),"scanned_gb":f"{scanned_gb:.1f}","scanned_lines":f"{scanned_lines_est//1000000}M", "speed":"~48 MB/s"})
        for fh in outs.values(): fh.close()
        files=[]
        for tgt in targets:
            path=os.path.join(out_dir, f"{tgt.replace('.','_')}.txt")
            if os.path.exists(path) and os.path.getsize(path)>0:
                name=f"{tgt}.txt"
                safe=re.sub(r'[^A-Za-z0-9._-]', '_', name)
                dl={}
                final=None
                # 1) persist on the Space's stable disk (survives restarts) + /tmp fallback
                try:
                    pd="/data/extracts"; os.makedirs(pd, exist_ok=True)
                    pd_path=os.path.join(pd, safe)
                    shutil.copy(path, pd_path)
                    final=pd_path
                    dl["download_url"]=f"/extract/download?file={safe}"
                except Exception as e:
                    print("persist fail:", e, flush=True)
                if final is None or (os.path.exists(final) and os.path.getsize(final)==0):
                    try:
                        td="/tmp/extracts"; os.makedirs(td, exist_ok=True)
                        tp=os.path.join(td, safe)
                        shutil.copy(path, tp); final=tp
                        dl["download_url"]=f"/extract/download?file={safe}"
                    except Exception as e:
                        print("tmp fail:", e, flush=True)
                if final is None:
                    final=f"/tmp/extract_{job_id}_{tgt.replace('.','_')}.txt"
                    shutil.copy(path, final)
                # 2) push into the Space repo files/ folder β€” DISABLED: causes Space rebuild which wipes jobs
                # if you want repo backup, re-enable below (will trigger a restart ~30s later)
                """
                try:
                    from huggingface_hub import HfApi as _HfApi
                    token=os.environ.get("HF_TOKEN") or os.environ.get("HF_TOKEN_WRITE")
                    if token and os.environ.get("HF_SPACE_ID"):
                        api=_HfApi(token=token)
                        api.upload_file(path_or_fileobj=path,
                                        path_in_repo=f"files/extracts/{safe}",
                                        repo_id=os.environ["HF_SPACE_ID"], repo_type="space")
                        dl["hf_url"]=(f"https://huggingface.co/spaces/{os.environ['HF_SPACE_ID']}/"
                                      f"resolve/main/files/extracts/{safe}?download=true")
                except Exception as e:
                    print("repo push fail:", e, flush=True)
                """
                files.append({"name":name,"lines":counts[tgt],"path":final, **dl})
            else:
                try: os.remove(path)
                except: pass
        try: os.remove(pat_path)
        except: pass
        try: shutil.rmtree(out_dir)
        except: pass
        jobs[job_id].update({"status":f"Done β€” {sum(counts.values())} lines","pct":100,"done":True,"files":files,"found":sum(counts.values())})
    except Exception as e:
        traceback.print_exc()
        jobs[job_id]={"status":f"Failed: {e}","pct":0,"done":True,"error":str(e)}

@app.post("/extract")
async def extract(request: Request):
    data=await request.json()
    targets=data.get("targets",[])
    if isinstance(targets, str):
        targets=[t.strip() for t in targets.split("\n") if t.strip()]
    targets=[t.strip() for t in targets if t.strip()]
    if not targets:
        return JSONResponse({"error":"Enter at least one keyword, one per line"}, status_code=400)
    if len(targets)>20:
        return JSONResponse({"error":"Max 20 keywords"}, status_code=400)
    selected=data.get("files")
    if isinstance(selected, str):
        selected=[s.strip() for s in selected.split("\n") if s.strip()]
    if isinstance(selected, list):
        selected=[s for s in selected if s and isinstance(s,str)]
        if not selected: selected=None
    job_id=uuid.uuid4().hex[:8]
    jobs[job_id]={"status":"Queued...","pct":0,"done":False}
    import threading
    threading.Thread(target=do_extract, args=(job_id, targets, selected), daemon=True).start()
    return {"job_id":job_id}

@app.get("/bucket/files")
async def bucket_files_list():
    try:
        return {"files": list_bucket_files()}
    except Exception as e:
        return {"error":str(e), "files":[]}

@app.get("/extract/progress")
async def extract_progress(job_id: str):
    j=jobs.get(job_id, {"status":"Unknown","pct":0})
    out={k:v for k,v in j.items() if not k.startswith("file_")}
    return out

@app.get("/extract/download")
async def extract_download(file: str):
    from fastapi.responses import Response
    try:
        safe=re.sub(r'[^A-Za-z0-9._-]', '_', file)
        found=None
        for base in ["/tmp/extracts", "/data/extracts", "/tmp", "/"]:
            p=os.path.join(base, safe) if base!="/" else f"/{safe}"
            try:
                if os.path.isfile(p): found=p; break
            except Exception:
                continue
        # fallback: glob any file starting with the target name
        if not found:
            import glob
            for pat in [f"/tmp/extract_*_{safe}", f"/tmp/extract_*_{safe.replace('.txt','')}_*.txt"]:
                m=glob.glob(pat)
                if m: found=m[0]; break
        if not found:
            return JSONResponse({"error":"File not ready or expired"}, status_code=404)
        with open(found,"rb") as fh:
            data=fh.read()
        return Response(content=data,
                        media_type="text/plain",
                        headers={"Content-Disposition":f'attachment; filename="{safe}"',
                                 "Cache-Control":"no-store"})
    except Exception as e:
        traceback.print_exc()
        return JSONResponse({"error":str(e)}, status_code=500)

@app.post("/fetch")
async def fetch_url(request: Request):
    data=await request.json()
    url=data.get("url","").strip()
    if not url or not url.startswith("http"):
        return JSONResponse({"status":"error","error":"Invalid URL"}, status_code=400)
    job_id=uuid.uuid4().hex[:8]
    jobs[job_id]={"status":"Queued...","pct":0}
    import threading
    threading.Thread(target=do_fetch, args=(job_id, url), daemon=True).start()
    return {"job_id":job_id,"status":"Started"}

@app.post("/fetch/gofile")
async def fetch_gofile(request: Request):
    data=await request.json()
    url=data.get("url","").strip()
    if not url or "gofile.io" not in url:
        return JSONResponse({"status":"error","error":"Invalid GoFile URL"}, status_code=400)
    job_id=uuid.uuid4().hex[:8]
    jobs[job_id]={"status":"Queued GoFile...","pct":0}
    import threading
    threading.Thread(target=do_gofile, args=(job_id, url), daemon=True).start()
    return {"job_id":job_id,"status":"Started"}

@app.post("/mediafire")
async def fetch_mediafire(request: Request):
    data=await request.json()
    url=data.get("url","").strip()
    if not url or ("mediafire.com" not in url and "mediafireapp.com" not in url):
        return JSONResponse({"status":"error","error":"Invalid MediaFire URL"}, status_code=400)
    job_id=uuid.uuid4().hex[:8]
    jobs[job_id]={"status":"Queued MediaFire...","pct":0}
    import threading
    threading.Thread(target=do_mediafire, args=(job_id, url), daemon=True).start()
    return {"job_id":job_id,"status":"Started"}

@app.get("/fetch/progress")
async def fetch_progress(job_id: str):
    return jobs.get(job_id, {"status":"Unknown job","pct":0})