| |
| """Acquire every hub-hosted source in the Dataset Provenance Register. |
| Writes an acquisition ledger (verified / failed) like ghdata/_reports/acquisition_ledger.json.""" |
| import os, json, time, traceback |
| from concurrent.futures import ThreadPoolExecutor, as_completed |
| os.environ.setdefault("HF_HOME", "/workspace/.hf_home") |
| from huggingface_hub import snapshot_download |
|
|
| ROOT = "/workspace/aditi/ghdata" |
| TOK = os.environ.get("HF_TOKEN") |
|
|
| |
| DATASETS = [ |
| |
| ("wandb/RAGTruth-processed","dataset","RAGTruth-processed"), |
| ("din0s/asqa","dataset","asqa"), |
| ("ai4bharat/Indic-Rag-Suite","dataset","Indic-Rag-Suite"), |
| ("pminervini/HaluEval","dataset","HaluEval"), |
| ("r-okamot/LLM-AggreFact_sub","dataset","LLM-AggreFact_sub"), |
| |
| ("utahnlp/x-fact","dataset","x-fact"), |
| ("Jezzarax/pubhealth-converted","dataset","pubhealth-converted"), |
| ("tdiggelm/climate_fever","dataset","climate_fever"), |
| ("pminervini/averitec","dataset","averitec"), |
| ("Dzeniks/feverous_3way","dataset","feverous_3way"), |
| ("chengxuphd/liar2","dataset","liar2"), |
| ("copenlu/fever_gold_evidence","dataset","fever_gold_evidence"), |
| ("Huggmachas/Stance_notebandi_dataset","dataset","Stance_notebandi"), |
| |
| ("nitishs32/HinFakeNews1","dataset","HinFakeNews1"), |
| ("community-datasets/urdu_fake_news","dataset","urdu_fake_news"), |
| ("hrshihab/BanFakeNews-2.0","dataset","BanFakeNews-2.0"), |
| ("Nithiwat/checkthat-binary","dataset","checkthat-binary"), |
| ("Nithiwat/claimbuster","dataset","claimbuster"), |
| ("LingoIITGN/COMI-LINGUA","dataset","COMI-LINGUA"), |
| ("Nhat243/IFND-multimodal","dataset","IFND-multimodal"), |
| |
| ("ai4bharat/naamapadam","dataset","naamapadam"), |
| ("Huggmachas/GLuecos_NER_EN_HI","dataset","GLuecos_NER_EN_HI"), |
| ("mirfan899/urdu-ner","dataset","urdu-ner"), |
| ("mirfan899/punjabi-ner","dataset","punjabi-ner"), |
| ("Huggmachas/Lince_benchmark_ner_hineng_train_valid","dataset","Lince_ner_hineng"), |
| ("unimelb-nlp/wikiann","dataset","wikiann"), |
| |
| ("cais/mmlu","dataset","mmlu"), |
| ("allenai/ai2_arc","dataset","ai2_arc"), |
| ("sarvamai/mmlu-indic","dataset","mmlu-indic"), |
| ("indicbench/arc_hi","dataset","arc_hi"), |
| |
| ("ai4bharat/IndicSentiment","dataset","IndicSentiment"), |
| ("sepidmnorozy/Urdu_sentiment","dataset","Urdu_sentiment"), |
| ("IsaacRodgz/DravidianCodeMix-Dataset","dataset","DravidianCodeMix"), |
| ("mteb/urdu_roman_sentiment","dataset","urdu_roman_sentiment"), |
| ("abhiprd2000/Maithili_Sentiment_8K","dataset","Maithili_Sentiment_8K"), |
| ("Baps24/odia-sentiment-dataset","dataset","odia-sentiment"), |
| ("Abhishek4896/hindi-english-code-mixed-tweets-sentiment","dataset","hinglish-tweet-sentiment"), |
| |
| ("nikitadesai/hasoc","dataset","hasoc"), |
| ("google/civil_comments","dataset","civil_comments"), |
| ("necrosyth/hindi-social-media-hostility-detection","dataset","constraint-hostility"), |
| ("ucberkeley-dlab/measuring-hate-speech","dataset","measuring-hate-speech"), |
| ("Hate-speech-CNERG/hatexplain","dataset","hatexplain"), |
| ("Huggmachas/aggression-dataset","dataset","trac-aggression"), |
| ("FrancophonIA/multilingual-hatespeech-dataset","dataset","multilingual-hatespeech"), |
| |
| ("liamdugan/raid","dataset","raid"), |
| ("anyangsong/COLING2025-MGT-Detection-Task1","dataset","COLING2025-MGT"), |
| ("AdoCleanCode/Fakeddit","dataset","Fakeddit"), |
| ("Ahren09/MMSoc_Memotion","dataset","Memotion"), |
| ("Eftekhar/BHM-Bengali-Hateful-Memes","dataset","BHM-Bengali-Memes"), |
| ("Anthony3456347095/MET-Meme","dataset","MET-Meme"), |
| ("mediabiasgroup/BABE","dataset","BABE"), |
| ("airt-ml/twitter-human-bots","dataset","twitter-human-bots"), |
| |
| ("ankurani/Factify5wqa","dataset","Factify5wqa"), |
| ("ai4bharat/Aksharantar","dataset","Aksharantar"), |
| ("nvidia/Daring-Anteater","dataset","Daring-Anteater"), |
| ("ai4bharat/indic-align","dataset","indic-align"), |
| ("allenai/tulu-3-sft-mixture","dataset","tulu-3-sft-mixture"), |
| ("sarvamai/samvaad-hi-v1","dataset","samvaad-hi-v1"), |
| ("CohereForAI/aya_dataset","dataset","aya_dataset"), |
| |
| ("MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7","model","judge-mdeberta-xnli"), |
| ] |
|
|
| def grab(item): |
| repo, rtype, name = item |
| dest = os.path.join(ROOT, name) |
| t0 = time.time() |
| try: |
| snapshot_download(repo_id=repo, repo_type=rtype, local_dir=dest, |
| max_workers=8, token=TOK) |
| sz = sum(os.path.getsize(os.path.join(dp,f)) |
| for dp,_,fs in os.walk(dest) for f in fs) |
| return {"repo":repo,"type":rtype,"local":name,"status":"verified", |
| "bytes":sz,"secs":round(time.time()-t0,1)} |
| except Exception as e: |
| return {"repo":repo,"type":rtype,"local":name,"status":"failed", |
| "error":f"{type(e).__name__}: {str(e)[:300]}","secs":round(time.time()-t0,1)} |
|
|
| if __name__ == "__main__": |
| os.makedirs(ROOT, exist_ok=True) |
| res=[] |
| with ThreadPoolExecutor(max_workers=10) as ex: |
| futs={ex.submit(grab,d):d for d in DATASETS} |
| for i,f in enumerate(as_completed(futs),1): |
| r=f.result(); res.append(r) |
| print(f"[{i}/{len(DATASETS)}] {r['status']:8s} {r['repo']} " |
| f"{r.get('bytes',0)/1e6:.0f}MB {r['secs']}s {r.get('error','')}", flush=True) |
| ok=[r for r in res if r['status']=='verified'] |
| os.makedirs("/workspace/aditi/ghdata/_reports", exist_ok=True) |
| json.dump({"generated":time.strftime("%Y-%m-%dT%H:%M:%SZ",time.gmtime()), |
| "verified":len(ok),"failed":len(res)-len(ok), |
| "total_bytes":sum(r.get('bytes',0) for r in ok),"items":res}, |
| open("/workspace/aditi/ghdata/_reports/acquisition_ledger.json","w"), indent=2) |
| print(f"\nLEDGER verified={len(ok)} failed={len(res)-len(ok)} " |
| f"bytes={sum(r.get('bytes',0) for r in ok)/1e9:.1f}GB", flush=True) |
|
|
