# (kind, ref, local_name, allow_patterns, licence) kind in {hf,git} # EVAL-ONLY sets are NOT here - contamination firewall (training on them destroys the gates). QUEUE=[ # ---------------- Tier A backbones (bounded samples only) ---------------- ("hf","ai4bharat/BPCC","bpcc",["*.md","*bpcc-seed*","*eng-hin*","*eng-ben*","*eng-urd*","*eng-tam*","*eng-kas*","*eng-snd*","*eng-brx*","*eng-sat*","*eng-mni*"],"CC0"), ("hf","uonlp/CulturaX","culturax_hi",["hi/hi_part_00000*","*.md"],"ODC-BY/mC4+OSCAR"), ("hf","allenai/MADLAD-400","madlad_indic",["data/hi/*0000*","data/ur/*0000*","data/bn/*0000*","*.md"],"ODC-BY"), ("hf","HuggingFaceFW/fineweb-2","fineweb2_indic",["data/hin_Deva/train/000_0000*","data/urd_Arab/train/000_0000*","*.md"],"ODC-BY"), # ---------------- Indic instruction / alignment ---------------- ("hf","ai4bharat/indic-instruct-data-v0.1","indic_instruct_v01",None,"CC-BY-4.0"), ("hf","CohereForAI/aya_dataset","aya_dataset",None,"Apache-2.0"), ("hf","sarvamai/samvaad-hi-v1","samvaad_hi",None,"CC-BY-4.0"), ("hf","nvidia/Daring-Anteater","daring_anteater2",None,"CC-BY-4.0"), ("hf","ai4bharat/indicvoices_r","indicvoices_r_txt",["*.json","*.csv","*.md"],"CC-BY-4.0"), # ---------------- NER / entity ---------------- ("hf","cfilt/HiNER-original","hiner_orig",None,"verify"), ("hf","ai4bharat/naamapadam","naamapadam2",None,"CC0"), ("git","https://github.com/AI4Bharat/IndicNER","IndicNER",None,"MIT"), ("git","https://github.com/AliWazir/SiNER-dataset","SiNER",None,"research"), # ---------------- misinformation / claim / fact-check ---------------- ("hf","Nithiwat/claimbuster","claimbuster3",None,"see repo"), ("hf","hrshihab/BanFakeNews-2.0","banfakenews3",None,"see repo"), ("hf","nitishs32/HinFakeNews1","hinfakenews2",None,"see repo"), ("git","https://github.com/THU-BPM/CHEF","CHEF",None,"research"), ("git","https://github.com/IKMLab/CFEVER-data","CFEVER",None,"research"), ("git","https://github.com/Tariq60/LIAR-PLUS","LIAR-PLUS",None,"CC0/see readme"), ("git","https://github.com/ramybaly/Article-Bias-Prediction","ArticleBias",None,"research"), ("hf","Jezzarax/pubhealth-converted","pubhealth2",None,"MIT"), ("hf","copenlu/fever_gold_evidence","fever_gold2",None,"CC-BY-SA-3.0"), ("hf","Dzeniks/feverous_3way","feverous2",None,"CC-BY-SA-3.0"), # ---------------- hostility / hate / abuse ---------------- ("git","https://github.com/ShareChatAI/MACD","MACD",None,"research"), ("git","https://github.com/haroonshakeel/roman_urdu_hate_speech","RUHSOLD",None,"MIT"), ("git","https://github.com/rezacsedu/Bengali-Hate-Speech-Dataset","BengaliHate",None,"MIT"), ("git","https://github.com/hate-alert/HateCheckHIn","HateCheckHIn",None,"CC-BY-4.0"), ("git","https://github.com/l3cube-pune/MarathiNLP","MarathiNLP",None,"CC-BY-NC-SA"), ("hf","Hate-speech-CNERG/hatexplain","hatexplain2",None,"CC-BY-4.0"), ("hf","ucberkeley-dlab/measuring-hate-speech","berkeley_hate2",None,"CC-BY-4.0"), ("hf","manueltonneau/india-hate-speech-superset","india_hate_superset",None,"research"), # ---------------- code-mix / LID / translit ---------------- ("git","https://github.com/l3cube-pune/code-mixed-nlp","l3cube_codemix",None,"CC-BY-NC-SA"), ("git","https://github.com/microsoft/GLUECoS","GLUECoS",None,"MIT"), ("git","https://github.com/google-research-datasets/dakshina","dakshina",None,"CC-BY-SA-4.0"), ("hf","ai4bharat/Aksharantar","aksharantar2",["*hin*","*ben*","*urd*","*tam*","*.md"],"CC0/verify"), # ---------------- topic / news ---------------- ("git","https://github.com/l3cube-pune/indic-nlp","l3cube_indicnlp",None,"CC-BY-NC-SA"), ("hf","rahular/varta","varta_small",["*small*","*.md"],"URLs/verify"), # ---------------- media / meme / synthetic-text ---------------- ("git","https://github.com/LCS2-IIITD/MOMENTA","MOMENTA",None,"research"), ("hf","liamdugan/raid","raid_small",["*test*","*.md"],"see repo"), ("hf","anyangsong/COLING2025-MGT-Detection-Task1","mgt2",None,"see repo"), ("hf","AdoCleanCode/Fakeddit","fakeddit2",None,"see repo"), # ---------------- grounding / RAG / hallucination ---------------- ("hf","ai4bharat/Indic-Rag-Suite","indic_rag2",["data/kas*","data/snd*","data/brx*","data/sat*","data/mni*","data/ne*","data/sa*","*.md"],"see repo"), ("hf","wandb/RAGTruth-processed","ragtruth2",None,"see repo"), ("hf","din0s/asqa","asqa2",None,"Apache-2.0"), # ---------------- general capability replay ---------------- ("hf","allenai/tulu-3-sft-mixture","tulu3_small",["data/train-0000[0-2]*","*.md"],"ODC-BY"), ("hf","open-thoughts/OpenThoughts3","openthoughts_small",["*0000[01]*","*.md"],"see repo"), ]