| |
| """Authenticated GitHub sweep. Discovers repos across every OSINT/Indic/multimodal axis |
| in the four dataset docs, ranks them, and writes a queue for the serial harvester.""" |
| import json,os,time,urllib.request,urllib.parse |
| TOK=os.environ["GITHUB_TOKEN"] |
| OUT="/workspace/aditi/ghdata/_reports/github_search.json" |
| def api(url): |
| req=urllib.request.Request(url,headers={"Authorization":f"Bearer {TOK}", |
| "User-Agent":"aditi","Accept":"application/vnd.github+json"}) |
| return json.load(urllib.request.urlopen(req,timeout=30)) |
| QUERIES=[ |
| |
| "hindi fake news dataset","urdu fake news dataset","bengali fake news dataset", |
| "indian fake news detection dataset","fact verification dataset","claim detection dataset", |
| "misinformation dataset multilingual","fact checking indic","rumour detection dataset", |
| "evidence retrieval fact verification","stance detection dataset","credibility news dataset", |
| |
| "hindi hate speech dataset","bengali hate speech dataset","urdu hate speech dataset", |
| "marathi hate speech","code-mixed hate speech","offensive language indic dataset", |
| "cyberbullying dataset indic","aggression detection dataset","toxic comment indic", |
| |
| "indic named entity recognition dataset","hindi NER dataset","urdu NER corpus", |
| "sindhi NER dataset","multilingual NER low resource","entity linking indian", |
| |
| "hinglish dataset","code mixed language identification","transliteration dataset indic", |
| "romanized indic dataset","code switching corpus", |
| |
| "multimodal hate meme dataset","meme dataset sentiment","video toxicity dataset", |
| "multimodal misinformation dataset","indic ocr dataset","document ai indic", |
| "image text dataset indian","deepfake detection dataset", |
| |
| "indic speech dataset asr","speech translation indic","audio abuse detection dataset", |
| "low resource speech corpus","code-mixed speech dataset", |
| |
| "kashmiri nlp","sindhi corpus nlp","pashto nlp dataset","tibetan nlp corpus", |
| "uyghur corpus nlp","bodo language dataset","santali language nlp","manipuri nlp dataset", |
| "maithili nlp","dogri language corpus", |
| |
| "indic instruction tuning dataset","indian language llm dataset","indic benchmark llm", |
| |
| "osint toolkit","social media analysis toolkit","open source intelligence framework", |
| ] |
| seen=json.load(open(OUT)) if os.path.exists(OUT) else {} |
| print(f"starting from {len(seen)} known repos",flush=True) |
| for i,q in enumerate(QUERIES,1): |
| for page in (1,2): |
| try: |
| u=("https://api.github.com/search/repositories?q="+urllib.parse.quote(q)+ |
| f"&sort=stars&order=desc&per_page=30&page={page}") |
| r=api(u) |
| for it in r.get("items",[]): |
| k=it["full_name"] |
| if k in seen: continue |
| seen[k]={"stars":it["stargazers_count"],"desc":(it.get("description") or "")[:150], |
| "url":it["clone_url"],"size_kb":it.get("size",0),"q":q, |
| "lang":it.get("language"),"pushed":it.get("pushed_at","")[:10]} |
| except Exception as e: |
| print(f"ERR {q} p{page}: {str(e)[:70]}",flush=True); break |
| time.sleep(2.2) |
| if i%10==0: print(f" [{i}/{len(QUERIES)}] {len(seen)} repos",flush=True) |
| json.dump(seen,open(OUT,"w"),indent=2) |
| print(f"\nTOTAL {len(seen)} distinct repos",flush=True) |
|
|
