code/sweep2.py

#!/usr/bin/env python
"""Sweep v2: exact HF repo ids + GitHub clones, from all four dataset documents.
Licence/firewall gating is explicit and enforced."""
import os,json,time,subprocess,shutil
from concurrent.futures import ThreadPoolExecutor, as_completed
os.environ.setdefault("HF_HOME","/workspace/.hf_home")
from huggingface_hub import snapshot_download
TOK=os.environ.get("HF_TOKEN"); OUT="/workspace/aditi/webdata"; GHD="/workspace/aditi/gh"
os.makedirs(OUT,exist_ok=True); os.makedirs(GHD,exist_ok=True)

HF=[  # (repo_id, local, allow_patterns or None)
 ("ai4bharat/IndicCorpV2","indiccorp_v2",["*hi*","*bn*","*ta*","*te*","*mr*","*gu*","*kn*","*ml*","*pa*","*or*","*as*","*ur*","*.json","*.md"]),
 ("ai4bharat/sangraha","sangraha",["verified/hin/*","verified/ben/*","verified/tam/*","verified/urd/*","*.md"]),
 ("CohereForAI/aya_collection","aya_collection",["*templated_indic*","*.md"]),
 ("manueltonneau/india-hate-speech-superset","india_hate_superset",None),
 ("cfilt/HiNER-original","hiner_original",None),
 ("cfilt/HiNER-collapsed","hiner_collapsed",None),
 ("ai4bharat/IN22-Gen","in22_gen_EVAL",None),
 ("bharatgenai/IndicParam","indicparam_EVAL",None),
 ("ai4bharat/MILU","milu_EVAL",None),
 ("haoranxu/FLORES-200","flores200_EVAL",None),
 ("hrshihab/BanFakeNews-2.0","banfakenews2",None),
 ("Nithiwat/claimbuster","claimbuster2",None),
 ("liamdugan/raid","raid2",None),
 ("haoranxu/ALMA-Human-Parallel","alma_parallel",None),
 ("uonlp/CulturaX","culturax",["*hi*","*bn*","*ur*","*ta*","*.md"]),
]
GIT=[ # (url, local)
 ("https://github.com/AI4Bharat/IndicLLMSuite","IndicLLMSuite"),
 ("https://github.com/l3cube-pune/indic-nlp","l3cube-indic-nlp"),
 ("https://github.com/l3cube-pune/MarathiNLP","l3cube-MarathiNLP"),
 ("https://github.com/l3cube-pune/code-mixed-nlp","l3cube-code-mixed-nlp"),
 ("https://github.com/ShareChatAI/MACD","MACD"),
 ("https://github.com/microsoft/GLUECoS","GLUECoS"),
 ("https://github.com/THU-BPM/CHEF","CHEF"),
 ("https://github.com/IKMLab/CFEVER-data","CFEVER-data"),
 ("https://github.com/AliWazir/SiNER-dataset","SiNER-dataset"),
 ("https://github.com/Sheetal83/Ax-to-Grind-Urdu-Dataset","Ax-to-Grind-Urdu"),
 ("https://github.com/haroonshakeel/roman_urdu_hate_speech","roman_urdu_hate_speech"),
 ("https://github.com/rezacsedu/Bengali-Hate-Speech-Dataset","Bengali-Hate-Speech"),
 ("https://github.com/hate-alert/HateCheckHIn","HateCheckHIn"),
 ("https://github.com/LCS2-IIITD/MOMENTA","MOMENTA"),
 ("https://github.com/ramybaly/Article-Bias-Prediction","Article-Bias-Prediction"),
 ("https://github.com/Tariq60/LIAR-PLUS","LIAR-PLUS"),
 ("https://github.com/google-research-datasets/dakshina","dakshina"),
 ("https://github.com/AI4Bharat/IndicBERT","IndicBERT"),
 ("https://github.com/AI4Bharat/indicnlp_catalog","indicnlp_catalog"),
 ("https://github.com/google-research-datasets/indic-gen-bench","indic-gen-bench_EVAL"),
]
# NEVER fetched into the training tree
FIREWALL={"FLORES-200":"NO-TRAIN (publisher) - fetched to *_EVAL only",
 "Bhasha-Abhijnaanam":"NO-TRAIN IndicLID benchmark","IndicGenBench":"Tier-6 eval firewall",
 "IndicParam":"EVAL-ONLY P4 regression tracker","MILU":"EVAL-ONLY",
 "IN22/FLORES":"EVAL-ONLY MT","IndicNLP Corpus":"EXCLUDED CC-BY-NC-SA",
 "Balochi/Saraiki (BECO, NAWA-E-WATAN, Jhoke)":"LICENCE-BLOCKED CC-BY-NC",
 "L3Cube MahaSent/MahaHate/MahaNER":"NC - owner approval required",
 "WikiANN/PAN-X":"SILVER - warm-start only, never eval",
 "MuMiN/TwiBot-22/TweetEval/GoEmotions/Higgs":"excluded per Handover Master"}

def hf_grab(it):
    repo,name,pats=it; t0=time.time(); dest=os.path.join(OUT,name)
    try:
        snapshot_download(repo_id=repo,repo_type="dataset",local_dir=dest,
                          allow_patterns=pats,max_workers=6,token=TOK)
        b=sum(os.path.getsize(os.path.join(d,f)) for d,_,fs in os.walk(dest) for f in fs)
        return {"src":"hf","repo":repo,"local":name,"status":"verified","bytes":b,"secs":round(time.time()-t0)}
    except Exception as e:
        k="gated" if "Gated" in type(e).__name__ or "403" in str(e) else "failed"
        return {"src":"hf","repo":repo,"status":k,"error":f"{type(e).__name__}: {str(e)[:120]}"}

def git_grab(it):
    url,name=it; t0=time.time(); dest=os.path.join(GHD,name)
    if os.path.isdir(dest): shutil.rmtree(dest,ignore_errors=True)
    try:
        r=subprocess.run(["git","clone","--depth","1","--quiet",url,dest],
                         capture_output=True,text=True,timeout=900)
        if r.returncode!=0:
            return {"src":"git","repo":url,"status":"failed","error":r.stderr[:160]}
        b=sum(os.path.getsize(os.path.join(d,f)) for d,_,fs in os.walk(dest)
              for f in fs if os.path.exists(os.path.join(d,f)))
        return {"src":"git","repo":url,"local":name,"status":"verified","bytes":b,"secs":round(time.time()-t0)}
    except Exception as e:
        return {"src":"git","repo":url,"status":"failed","error":str(e)[:160]}

if __name__=="__main__":
    res=[]
    with ThreadPoolExecutor(max_workers=8) as ex:
        futs=[ex.submit(hf_grab,i) for i in HF]+[ex.submit(git_grab,i) for i in GIT]
        for i,f in enumerate(as_completed(futs),1):
            r=f.result(); res.append(r)
            print(f"[{i}/{len(futs)}] {r['status']:9s} {r['src']:3s} {r['repo'][-56:]:56s} "
                  f"{r.get('bytes',0)/1e6:8.0f}MB {r.get('error','')[:60]}",flush=True)
    ok=[r for r in res if r['status']=='verified']
    gated=[r for r in res if r['status']=='gated']
    json.dump({"generated":time.strftime("%Y-%m-%dT%H:%M:%SZ",time.gmtime()),
        "verified":len(ok),"gated":len(gated),"failed":len(res)-len(ok)-len(gated),
        "bytes":sum(r.get('bytes',0) for r in ok),"firewall":FIREWALL,"items":res},
        open("/workspace/aditi/ghdata/_reports/sweep2_ledger.json","w"),indent=2)
    print(f"\nSWEEP2 verified={len(ok)} gated={len(gated)} failed={len(res)-len(ok)-len(gated)} "
          f"{sum(r.get('bytes',0) for r in ok)/1e9:.1f}GB",flush=True)
    for g in gated: print("  GATED (needs licence acceptance):",g['repo'])