File size: 4,608 Bytes
e7d8c8b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | # (kind, ref, local_name, allow_patterns, licence) kind in {hf,git}
# EVAL-ONLY sets are NOT here - contamination firewall (training on them destroys the gates).
QUEUE=[
# ---------------- Tier A backbones (bounded samples only) ----------------
("hf","ai4bharat/BPCC","bpcc",["*.md","*bpcc-seed*","*eng-hin*","*eng-ben*","*eng-urd*","*eng-tam*","*eng-kas*","*eng-snd*","*eng-brx*","*eng-sat*","*eng-mni*"],"CC0"),
("hf","uonlp/CulturaX","culturax_hi",["hi/hi_part_00000*","*.md"],"ODC-BY/mC4+OSCAR"),
("hf","allenai/MADLAD-400","madlad_indic",["data/hi/*0000*","data/ur/*0000*","data/bn/*0000*","*.md"],"ODC-BY"),
("hf","HuggingFaceFW/fineweb-2","fineweb2_indic",["data/hin_Deva/train/000_0000*","data/urd_Arab/train/000_0000*","*.md"],"ODC-BY"),
# ---------------- Indic instruction / alignment ----------------
("hf","ai4bharat/indic-instruct-data-v0.1","indic_instruct_v01",None,"CC-BY-4.0"),
("hf","CohereForAI/aya_dataset","aya_dataset",None,"Apache-2.0"),
("hf","sarvamai/samvaad-hi-v1","samvaad_hi",None,"CC-BY-4.0"),
("hf","nvidia/Daring-Anteater","daring_anteater2",None,"CC-BY-4.0"),
("hf","ai4bharat/indicvoices_r","indicvoices_r_txt",["*.json","*.csv","*.md"],"CC-BY-4.0"),
# ---------------- NER / entity ----------------
("hf","cfilt/HiNER-original","hiner_orig",None,"verify"),
("hf","ai4bharat/naamapadam","naamapadam2",None,"CC0"),
("git","https://github.com/AI4Bharat/IndicNER","IndicNER",None,"MIT"),
("git","https://github.com/AliWazir/SiNER-dataset","SiNER",None,"research"),
# ---------------- misinformation / claim / fact-check ----------------
("hf","Nithiwat/claimbuster","claimbuster3",None,"see repo"),
("hf","hrshihab/BanFakeNews-2.0","banfakenews3",None,"see repo"),
("hf","nitishs32/HinFakeNews1","hinfakenews2",None,"see repo"),
("git","https://github.com/THU-BPM/CHEF","CHEF",None,"research"),
("git","https://github.com/IKMLab/CFEVER-data","CFEVER",None,"research"),
("git","https://github.com/Tariq60/LIAR-PLUS","LIAR-PLUS",None,"CC0/see readme"),
("git","https://github.com/ramybaly/Article-Bias-Prediction","ArticleBias",None,"research"),
("hf","Jezzarax/pubhealth-converted","pubhealth2",None,"MIT"),
("hf","copenlu/fever_gold_evidence","fever_gold2",None,"CC-BY-SA-3.0"),
("hf","Dzeniks/feverous_3way","feverous2",None,"CC-BY-SA-3.0"),
# ---------------- hostility / hate / abuse ----------------
("git","https://github.com/ShareChatAI/MACD","MACD",None,"research"),
("git","https://github.com/haroonshakeel/roman_urdu_hate_speech","RUHSOLD",None,"MIT"),
("git","https://github.com/rezacsedu/Bengali-Hate-Speech-Dataset","BengaliHate",None,"MIT"),
("git","https://github.com/hate-alert/HateCheckHIn","HateCheckHIn",None,"CC-BY-4.0"),
("git","https://github.com/l3cube-pune/MarathiNLP","MarathiNLP",None,"CC-BY-NC-SA"),
("hf","Hate-speech-CNERG/hatexplain","hatexplain2",None,"CC-BY-4.0"),
("hf","ucberkeley-dlab/measuring-hate-speech","berkeley_hate2",None,"CC-BY-4.0"),
("hf","manueltonneau/india-hate-speech-superset","india_hate_superset",None,"research"),
# ---------------- code-mix / LID / translit ----------------
("git","https://github.com/l3cube-pune/code-mixed-nlp","l3cube_codemix",None,"CC-BY-NC-SA"),
("git","https://github.com/microsoft/GLUECoS","GLUECoS",None,"MIT"),
("git","https://github.com/google-research-datasets/dakshina","dakshina",None,"CC-BY-SA-4.0"),
("hf","ai4bharat/Aksharantar","aksharantar2",["*hin*","*ben*","*urd*","*tam*","*.md"],"CC0/verify"),
# ---------------- topic / news ----------------
("git","https://github.com/l3cube-pune/indic-nlp","l3cube_indicnlp",None,"CC-BY-NC-SA"),
("hf","rahular/varta","varta_small",["*small*","*.md"],"URLs/verify"),
# ---------------- media / meme / synthetic-text ----------------
("git","https://github.com/LCS2-IIITD/MOMENTA","MOMENTA",None,"research"),
("hf","liamdugan/raid","raid_small",["*test*","*.md"],"see repo"),
("hf","anyangsong/COLING2025-MGT-Detection-Task1","mgt2",None,"see repo"),
("hf","AdoCleanCode/Fakeddit","fakeddit2",None,"see repo"),
# ---------------- grounding / RAG / hallucination ----------------
("hf","ai4bharat/Indic-Rag-Suite","indic_rag2",["data/kas*","data/snd*","data/brx*","data/sat*","data/mni*","data/ne*","data/sa*","*.md"],"see repo"),
("hf","wandb/RAGTruth-processed","ragtruth2",None,"see repo"),
("hf","din0s/asqa","asqa2",None,"Apache-2.0"),
# ---------------- general capability replay ----------------
("hf","allenai/tulu-3-sft-mixture","tulu3_small",["data/train-0000[0-2]*","*.md"],"ODC-BY"),
("hf","open-thoughts/OpenThoughts3","openthoughts_small",["*0000[01]*","*.md"],"see repo"),
]
|