File size: 6,095 Bytes
e7d8c8b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 | #!/usr/bin/env python
"""Acquire every hub-hosted source in the Dataset Provenance Register.
Writes an acquisition ledger (verified / failed) like ghdata/_reports/acquisition_ledger.json."""
import os, json, time, traceback
from concurrent.futures import ThreadPoolExecutor, as_completed
os.environ.setdefault("HF_HOME", "/workspace/.hf_home")
from huggingface_hub import snapshot_download
ROOT = "/workspace/aditi/ghdata"
TOK = os.environ.get("HF_TOKEN")
# (repo_id, repo_type, local_name) -- from Provenance Register §1,§2 + Handover §2
DATASETS = [
# --- faithful_gen / halluc / halluc_indic (G1+G2, 34% of compute) ---
("wandb/RAGTruth-processed","dataset","RAGTruth-processed"),
("din0s/asqa","dataset","asqa"),
("ai4bharat/Indic-Rag-Suite","dataset","Indic-Rag-Suite"),
("pminervini/HaluEval","dataset","HaluEval"),
("r-okamot/LLM-AggreFact_sub","dataset","LLM-AggreFact_sub"),
# --- claim / stance (G1) ---
("utahnlp/x-fact","dataset","x-fact"),
("Jezzarax/pubhealth-converted","dataset","pubhealth-converted"),
("tdiggelm/climate_fever","dataset","climate_fever"),
("pminervini/averitec","dataset","averitec"),
("Dzeniks/feverous_3way","dataset","feverous_3way"),
("chengxuphd/liar2","dataset","liar2"),
("copenlu/fever_gold_evidence","dataset","fever_gold_evidence"),
("Huggmachas/Stance_notebandi_dataset","dataset","Stance_notebandi"),
# --- schema_lock (G3) ---
("nitishs32/HinFakeNews1","dataset","HinFakeNews1"),
("community-datasets/urdu_fake_news","dataset","urdu_fake_news"),
("hrshihab/BanFakeNews-2.0","dataset","BanFakeNews-2.0"),
("Nithiwat/checkthat-binary","dataset","checkthat-binary"),
("Nithiwat/claimbuster","dataset","claimbuster"),
("LingoIITGN/COMI-LINGUA","dataset","COMI-LINGUA"),
("Nhat243/IFND-multimodal","dataset","IFND-multimodal"),
# --- ner / geo ---
("ai4bharat/naamapadam","dataset","naamapadam"),
("Huggmachas/GLuecos_NER_EN_HI","dataset","GLuecos_NER_EN_HI"),
("mirfan899/urdu-ner","dataset","urdu-ner"),
("mirfan899/punjabi-ner","dataset","punjabi-ner"),
("Huggmachas/Lince_benchmark_ner_hineng_train_valid","dataset","Lince_ner_hineng"),
("unimelb-nlp/wikiann","dataset","wikiann"),
# --- general (G4 no-regression) ---
("cais/mmlu","dataset","mmlu"),
("allenai/ai2_arc","dataset","ai2_arc"),
("sarvamai/mmlu-indic","dataset","mmlu-indic"),
("indicbench/arc_hi","dataset","arc_hi"),
# --- sentiment ---
("ai4bharat/IndicSentiment","dataset","IndicSentiment"),
("sepidmnorozy/Urdu_sentiment","dataset","Urdu_sentiment"),
("IsaacRodgz/DravidianCodeMix-Dataset","dataset","DravidianCodeMix"),
("mteb/urdu_roman_sentiment","dataset","urdu_roman_sentiment"),
("abhiprd2000/Maithili_Sentiment_8K","dataset","Maithili_Sentiment_8K"),
("Baps24/odia-sentiment-dataset","dataset","odia-sentiment"),
("Abhishek4896/hindi-english-code-mixed-tweets-sentiment","dataset","hinglish-tweet-sentiment"),
# --- hostility ---
("nikitadesai/hasoc","dataset","hasoc"),
("google/civil_comments","dataset","civil_comments"),
("necrosyth/hindi-social-media-hostility-detection","dataset","constraint-hostility"),
("ucberkeley-dlab/measuring-hate-speech","dataset","measuring-hate-speech"),
("Hate-speech-CNERG/hatexplain","dataset","hatexplain"),
("Huggmachas/aggression-dataset","dataset","trac-aggression"),
("FrancophonIA/multilingual-hatespeech-dataset","dataset","multilingual-hatespeech"),
# --- media / meme / credibility ---
("liamdugan/raid","dataset","raid"),
("anyangsong/COLING2025-MGT-Detection-Task1","dataset","COLING2025-MGT"),
("AdoCleanCode/Fakeddit","dataset","Fakeddit"),
("Ahren09/MMSoc_Memotion","dataset","Memotion"),
("Eftekhar/BHM-Bengali-Hateful-Memes","dataset","BHM-Bengali-Memes"),
("Anthony3456347095/MET-Meme","dataset","MET-Meme"),
("mediabiasgroup/BABE","dataset","BABE"),
("airt-ml/twitter-human-bots","dataset","twitter-human-bots"),
# --- report / topic / translit / replay ---
("ankurani/Factify5wqa","dataset","Factify5wqa"),
("ai4bharat/Aksharantar","dataset","Aksharantar"),
("nvidia/Daring-Anteater","dataset","Daring-Anteater"),
("ai4bharat/indic-align","dataset","indic-align"),
("allenai/tulu-3-sft-mixture","dataset","tulu-3-sft-mixture"),
("sarvamai/samvaad-hi-v1","dataset","samvaad-hi-v1"),
("CohereForAI/aya_dataset","dataset","aya_dataset"),
# --- G2 judge ---
("MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7","model","judge-mdeberta-xnli"),
]
def grab(item):
repo, rtype, name = item
dest = os.path.join(ROOT, name)
t0 = time.time()
try:
snapshot_download(repo_id=repo, repo_type=rtype, local_dir=dest,
max_workers=8, token=TOK)
sz = sum(os.path.getsize(os.path.join(dp,f))
for dp,_,fs in os.walk(dest) for f in fs)
return {"repo":repo,"type":rtype,"local":name,"status":"verified",
"bytes":sz,"secs":round(time.time()-t0,1)}
except Exception as e:
return {"repo":repo,"type":rtype,"local":name,"status":"failed",
"error":f"{type(e).__name__}: {str(e)[:300]}","secs":round(time.time()-t0,1)}
if __name__ == "__main__":
os.makedirs(ROOT, exist_ok=True)
res=[]
with ThreadPoolExecutor(max_workers=10) as ex:
futs={ex.submit(grab,d):d for d in DATASETS}
for i,f in enumerate(as_completed(futs),1):
r=f.result(); res.append(r)
print(f"[{i}/{len(DATASETS)}] {r['status']:8s} {r['repo']} "
f"{r.get('bytes',0)/1e6:.0f}MB {r['secs']}s {r.get('error','')}", flush=True)
ok=[r for r in res if r['status']=='verified']
os.makedirs("/workspace/aditi/ghdata/_reports", exist_ok=True)
json.dump({"generated":time.strftime("%Y-%m-%dT%H:%M:%SZ",time.gmtime()),
"verified":len(ok),"failed":len(res)-len(ok),
"total_bytes":sum(r.get('bytes',0) for r in ok),"items":res},
open("/workspace/aditi/ghdata/_reports/acquisition_ledger.json","w"), indent=2)
print(f"\nLEDGER verified={len(ok)} failed={len(res)-len(ok)} "
f"bytes={sum(r.get('bytes',0) for r in ok)/1e9:.1f}GB", flush=True)
|