#!/usr/bin/env python
"""Acquire every hub-hosted source in the Dataset Provenance Register.
Writes an acquisition ledger (verified / failed) like ghdata/_reports/acquisition_ledger.json."""
import os, json, time, traceback
from concurrent.futures import ThreadPoolExecutor, as_completed
os.environ.setdefault("HF_HOME", "/workspace/.hf_home")
from huggingface_hub import snapshot_download

ROOT = "/workspace/aditi/ghdata"
TOK  = os.environ.get("HF_TOKEN")

# (repo_id, repo_type, local_name)  -- from Provenance Register §1,§2 + Handover §2
DATASETS = [
 # --- faithful_gen / halluc / halluc_indic (G1+G2, 34% of compute) ---
 ("wandb/RAGTruth-processed","dataset","RAGTruth-processed"),
 ("din0s/asqa","dataset","asqa"),
 ("ai4bharat/Indic-Rag-Suite","dataset","Indic-Rag-Suite"),
 ("pminervini/HaluEval","dataset","HaluEval"),
 ("r-okamot/LLM-AggreFact_sub","dataset","LLM-AggreFact_sub"),
 # --- claim / stance (G1) ---
 ("utahnlp/x-fact","dataset","x-fact"),
 ("Jezzarax/pubhealth-converted","dataset","pubhealth-converted"),
 ("tdiggelm/climate_fever","dataset","climate_fever"),
 ("pminervini/averitec","dataset","averitec"),
 ("Dzeniks/feverous_3way","dataset","feverous_3way"),
 ("chengxuphd/liar2","dataset","liar2"),
 ("copenlu/fever_gold_evidence","dataset","fever_gold_evidence"),
 ("Huggmachas/Stance_notebandi_dataset","dataset","Stance_notebandi"),
 # --- schema_lock (G3) ---
 ("nitishs32/HinFakeNews1","dataset","HinFakeNews1"),
 ("community-datasets/urdu_fake_news","dataset","urdu_fake_news"),
 ("hrshihab/BanFakeNews-2.0","dataset","BanFakeNews-2.0"),
 ("Nithiwat/checkthat-binary","dataset","checkthat-binary"),
 ("Nithiwat/claimbuster","dataset","claimbuster"),
 ("LingoIITGN/COMI-LINGUA","dataset","COMI-LINGUA"),
 ("Nhat243/IFND-multimodal","dataset","IFND-multimodal"),
 # --- ner / geo ---
 ("ai4bharat/naamapadam","dataset","naamapadam"),
 ("Huggmachas/GLuecos_NER_EN_HI","dataset","GLuecos_NER_EN_HI"),
 ("mirfan899/urdu-ner","dataset","urdu-ner"),
 ("mirfan899/punjabi-ner","dataset","punjabi-ner"),
 ("Huggmachas/Lince_benchmark_ner_hineng_train_valid","dataset","Lince_ner_hineng"),
 ("unimelb-nlp/wikiann","dataset","wikiann"),
 # --- general (G4 no-regression) ---
 ("cais/mmlu","dataset","mmlu"),
 ("allenai/ai2_arc","dataset","ai2_arc"),
 ("sarvamai/mmlu-indic","dataset","mmlu-indic"),
 ("indicbench/arc_hi","dataset","arc_hi"),
 # --- sentiment ---
 ("ai4bharat/IndicSentiment","dataset","IndicSentiment"),
 ("sepidmnorozy/Urdu_sentiment","dataset","Urdu_sentiment"),
 ("IsaacRodgz/DravidianCodeMix-Dataset","dataset","DravidianCodeMix"),
 ("mteb/urdu_roman_sentiment","dataset","urdu_roman_sentiment"),
 ("abhiprd2000/Maithili_Sentiment_8K","dataset","Maithili_Sentiment_8K"),
 ("Baps24/odia-sentiment-dataset","dataset","odia-sentiment"),
 ("Abhishek4896/hindi-english-code-mixed-tweets-sentiment","dataset","hinglish-tweet-sentiment"),
 # --- hostility ---
 ("nikitadesai/hasoc","dataset","hasoc"),
 ("google/civil_comments","dataset","civil_comments"),
 ("necrosyth/hindi-social-media-hostility-detection","dataset","constraint-hostility"),
 ("ucberkeley-dlab/measuring-hate-speech","dataset","measuring-hate-speech"),
 ("Hate-speech-CNERG/hatexplain","dataset","hatexplain"),
 ("Huggmachas/aggression-dataset","dataset","trac-aggression"),
 ("FrancophonIA/multilingual-hatespeech-dataset","dataset","multilingual-hatespeech"),
 # --- media / meme / credibility ---
 ("liamdugan/raid","dataset","raid"),
 ("anyangsong/COLING2025-MGT-Detection-Task1","dataset","COLING2025-MGT"),
 ("AdoCleanCode/Fakeddit","dataset","Fakeddit"),
 ("Ahren09/MMSoc_Memotion","dataset","Memotion"),
 ("Eftekhar/BHM-Bengali-Hateful-Memes","dataset","BHM-Bengali-Memes"),
 ("Anthony3456347095/MET-Meme","dataset","MET-Meme"),
 ("mediabiasgroup/BABE","dataset","BABE"),
 ("airt-ml/twitter-human-bots","dataset","twitter-human-bots"),
 # --- report / topic / translit / replay ---
 ("ankurani/Factify5wqa","dataset","Factify5wqa"),
 ("ai4bharat/Aksharantar","dataset","Aksharantar"),
 ("nvidia/Daring-Anteater","dataset","Daring-Anteater"),
 ("ai4bharat/indic-align","dataset","indic-align"),
 ("allenai/tulu-3-sft-mixture","dataset","tulu-3-sft-mixture"),
 ("sarvamai/samvaad-hi-v1","dataset","samvaad-hi-v1"),
 ("CohereForAI/aya_dataset","dataset","aya_dataset"),
 # --- G2 judge ---
 ("MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7","model","judge-mdeberta-xnli"),
]

def grab(item):
    repo, rtype, name = item
    dest = os.path.join(ROOT, name)
    t0 = time.time()
    try:
        snapshot_download(repo_id=repo, repo_type=rtype, local_dir=dest,
                          max_workers=8, token=TOK)
        sz = sum(os.path.getsize(os.path.join(dp,f))
                 for dp,_,fs in os.walk(dest) for f in fs)
        return {"repo":repo,"type":rtype,"local":name,"status":"verified",
                "bytes":sz,"secs":round(time.time()-t0,1)}
    except Exception as e:
        return {"repo":repo,"type":rtype,"local":name,"status":"failed",
                "error":f"{type(e).__name__}: {str(e)[:300]}","secs":round(time.time()-t0,1)}

if __name__ == "__main__":
    os.makedirs(ROOT, exist_ok=True)
    res=[]
    with ThreadPoolExecutor(max_workers=10) as ex:
        futs={ex.submit(grab,d):d for d in DATASETS}
        for i,f in enumerate(as_completed(futs),1):
            r=f.result(); res.append(r)
            print(f"[{i}/{len(DATASETS)}] {r['status']:8s} {r['repo']} "
                  f"{r.get('bytes',0)/1e6:.0f}MB {r['secs']}s {r.get('error','')}", flush=True)
    ok=[r for r in res if r['status']=='verified']
    os.makedirs("/workspace/aditi/ghdata/_reports", exist_ok=True)
    json.dump({"generated":time.strftime("%Y-%m-%dT%H:%M:%SZ",time.gmtime()),
               "verified":len(ok),"failed":len(res)-len(ok),
               "total_bytes":sum(r.get('bytes',0) for r in ok),"items":res},
              open("/workspace/aditi/ghdata/_reports/acquisition_ledger.json","w"), indent=2)
    print(f"\nLEDGER verified={len(ok)} failed={len(res)-len(ok)} "
          f"bytes={sum(r.get('bytes',0) for r in ok)/1e9:.1f}GB", flush=True)