| |
| """Sweep v2: exact HF repo ids + GitHub clones, from all four dataset documents. |
| Licence/firewall gating is explicit and enforced.""" |
| import os,json,time,subprocess,shutil |
| from concurrent.futures import ThreadPoolExecutor, as_completed |
| os.environ.setdefault("HF_HOME","/workspace/.hf_home") |
| from huggingface_hub import snapshot_download |
| TOK=os.environ.get("HF_TOKEN"); OUT="/workspace/aditi/webdata"; GHD="/workspace/aditi/gh" |
| os.makedirs(OUT,exist_ok=True); os.makedirs(GHD,exist_ok=True) |
|
|
| HF=[ |
| ("ai4bharat/IndicCorpV2","indiccorp_v2",["*hi*","*bn*","*ta*","*te*","*mr*","*gu*","*kn*","*ml*","*pa*","*or*","*as*","*ur*","*.json","*.md"]), |
| ("ai4bharat/sangraha","sangraha",["verified/hin/*","verified/ben/*","verified/tam/*","verified/urd/*","*.md"]), |
| ("CohereForAI/aya_collection","aya_collection",["*templated_indic*","*.md"]), |
| ("manueltonneau/india-hate-speech-superset","india_hate_superset",None), |
| ("cfilt/HiNER-original","hiner_original",None), |
| ("cfilt/HiNER-collapsed","hiner_collapsed",None), |
| ("ai4bharat/IN22-Gen","in22_gen_EVAL",None), |
| ("bharatgenai/IndicParam","indicparam_EVAL",None), |
| ("ai4bharat/MILU","milu_EVAL",None), |
| ("haoranxu/FLORES-200","flores200_EVAL",None), |
| ("hrshihab/BanFakeNews-2.0","banfakenews2",None), |
| ("Nithiwat/claimbuster","claimbuster2",None), |
| ("liamdugan/raid","raid2",None), |
| ("haoranxu/ALMA-Human-Parallel","alma_parallel",None), |
| ("uonlp/CulturaX","culturax",["*hi*","*bn*","*ur*","*ta*","*.md"]), |
| ] |
| GIT=[ |
| ("https://github.com/AI4Bharat/IndicLLMSuite","IndicLLMSuite"), |
| ("https://github.com/l3cube-pune/indic-nlp","l3cube-indic-nlp"), |
| ("https://github.com/l3cube-pune/MarathiNLP","l3cube-MarathiNLP"), |
| ("https://github.com/l3cube-pune/code-mixed-nlp","l3cube-code-mixed-nlp"), |
| ("https://github.com/ShareChatAI/MACD","MACD"), |
| ("https://github.com/microsoft/GLUECoS","GLUECoS"), |
| ("https://github.com/THU-BPM/CHEF","CHEF"), |
| ("https://github.com/IKMLab/CFEVER-data","CFEVER-data"), |
| ("https://github.com/AliWazir/SiNER-dataset","SiNER-dataset"), |
| ("https://github.com/Sheetal83/Ax-to-Grind-Urdu-Dataset","Ax-to-Grind-Urdu"), |
| ("https://github.com/haroonshakeel/roman_urdu_hate_speech","roman_urdu_hate_speech"), |
| ("https://github.com/rezacsedu/Bengali-Hate-Speech-Dataset","Bengali-Hate-Speech"), |
| ("https://github.com/hate-alert/HateCheckHIn","HateCheckHIn"), |
| ("https://github.com/LCS2-IIITD/MOMENTA","MOMENTA"), |
| ("https://github.com/ramybaly/Article-Bias-Prediction","Article-Bias-Prediction"), |
| ("https://github.com/Tariq60/LIAR-PLUS","LIAR-PLUS"), |
| ("https://github.com/google-research-datasets/dakshina","dakshina"), |
| ("https://github.com/AI4Bharat/IndicBERT","IndicBERT"), |
| ("https://github.com/AI4Bharat/indicnlp_catalog","indicnlp_catalog"), |
| ("https://github.com/google-research-datasets/indic-gen-bench","indic-gen-bench_EVAL"), |
| ] |
| |
| FIREWALL={"FLORES-200":"NO-TRAIN (publisher) - fetched to *_EVAL only", |
| "Bhasha-Abhijnaanam":"NO-TRAIN IndicLID benchmark","IndicGenBench":"Tier-6 eval firewall", |
| "IndicParam":"EVAL-ONLY P4 regression tracker","MILU":"EVAL-ONLY", |
| "IN22/FLORES":"EVAL-ONLY MT","IndicNLP Corpus":"EXCLUDED CC-BY-NC-SA", |
| "Balochi/Saraiki (BECO, NAWA-E-WATAN, Jhoke)":"LICENCE-BLOCKED CC-BY-NC", |
| "L3Cube MahaSent/MahaHate/MahaNER":"NC - owner approval required", |
| "WikiANN/PAN-X":"SILVER - warm-start only, never eval", |
| "MuMiN/TwiBot-22/TweetEval/GoEmotions/Higgs":"excluded per Handover Master"} |
|
|
| def hf_grab(it): |
| repo,name,pats=it; t0=time.time(); dest=os.path.join(OUT,name) |
| try: |
| snapshot_download(repo_id=repo,repo_type="dataset",local_dir=dest, |
| allow_patterns=pats,max_workers=6,token=TOK) |
| b=sum(os.path.getsize(os.path.join(d,f)) for d,_,fs in os.walk(dest) for f in fs) |
| return {"src":"hf","repo":repo,"local":name,"status":"verified","bytes":b,"secs":round(time.time()-t0)} |
| except Exception as e: |
| k="gated" if "Gated" in type(e).__name__ or "403" in str(e) else "failed" |
| return {"src":"hf","repo":repo,"status":k,"error":f"{type(e).__name__}: {str(e)[:120]}"} |
|
|
| def git_grab(it): |
| url,name=it; t0=time.time(); dest=os.path.join(GHD,name) |
| if os.path.isdir(dest): shutil.rmtree(dest,ignore_errors=True) |
| try: |
| r=subprocess.run(["git","clone","--depth","1","--quiet",url,dest], |
| capture_output=True,text=True,timeout=900) |
| if r.returncode!=0: |
| return {"src":"git","repo":url,"status":"failed","error":r.stderr[:160]} |
| b=sum(os.path.getsize(os.path.join(d,f)) for d,_,fs in os.walk(dest) |
| for f in fs if os.path.exists(os.path.join(d,f))) |
| return {"src":"git","repo":url,"local":name,"status":"verified","bytes":b,"secs":round(time.time()-t0)} |
| except Exception as e: |
| return {"src":"git","repo":url,"status":"failed","error":str(e)[:160]} |
|
|
| if __name__=="__main__": |
| res=[] |
| with ThreadPoolExecutor(max_workers=8) as ex: |
| futs=[ex.submit(hf_grab,i) for i in HF]+[ex.submit(git_grab,i) for i in GIT] |
| for i,f in enumerate(as_completed(futs),1): |
| r=f.result(); res.append(r) |
| print(f"[{i}/{len(futs)}] {r['status']:9s} {r['src']:3s} {r['repo'][-56:]:56s} " |
| f"{r.get('bytes',0)/1e6:8.0f}MB {r.get('error','')[:60]}",flush=True) |
| ok=[r for r in res if r['status']=='verified'] |
| gated=[r for r in res if r['status']=='gated'] |
| json.dump({"generated":time.strftime("%Y-%m-%dT%H:%M:%SZ",time.gmtime()), |
| "verified":len(ok),"gated":len(gated),"failed":len(res)-len(ok)-len(gated), |
| "bytes":sum(r.get('bytes',0) for r in ok),"firewall":FIREWALL,"items":res}, |
| open("/workspace/aditi/ghdata/_reports/sweep2_ledger.json","w"),indent=2) |
| print(f"\nSWEEP2 verified={len(ok)} gated={len(gated)} failed={len(res)-len(ok)-len(gated)} " |
| f"{sum(r.get('bytes',0) for r in ok)/1e9:.1f}GB",flush=True) |
| for g in gated: print(" GATED (needs licence acceptance):",g['repo']) |
|
|
