#!/usr/bin/env python
"""Authenticated GitHub sweep. Discovers repos across every OSINT/Indic/multimodal axis
in the four dataset docs, ranks them, and writes a queue for the serial harvester."""
import json,os,time,urllib.request,urllib.parse
TOK=os.environ["GITHUB_TOKEN"]
OUT="/workspace/aditi/ghdata/_reports/github_search.json"
def api(url):
    req=urllib.request.Request(url,headers={"Authorization":f"Bearer {TOK}",
        "User-Agent":"aditi","Accept":"application/vnd.github+json"})
    return json.load(urllib.request.urlopen(req,timeout=30))
QUERIES=[
 # --- misinformation / fact-check ---
 "hindi fake news dataset","urdu fake news dataset","bengali fake news dataset",
 "indian fake news detection dataset","fact verification dataset","claim detection dataset",
 "misinformation dataset multilingual","fact checking indic","rumour detection dataset",
 "evidence retrieval fact verification","stance detection dataset","credibility news dataset",
 # --- hate / hostility / abuse ---
 "hindi hate speech dataset","bengali hate speech dataset","urdu hate speech dataset",
 "marathi hate speech","code-mixed hate speech","offensive language indic dataset",
 "cyberbullying dataset indic","aggression detection dataset","toxic comment indic",
 # --- NER / entity / IE ---
 "indic named entity recognition dataset","hindi NER dataset","urdu NER corpus",
 "sindhi NER dataset","multilingual NER low resource","entity linking indian",
 # --- code-mix / LID / translit ---
 "hinglish dataset","code mixed language identification","transliteration dataset indic",
 "romanized indic dataset","code switching corpus",
 # --- multimodal / vision / video ---
 "multimodal hate meme dataset","meme dataset sentiment","video toxicity dataset",
 "multimodal misinformation dataset","indic ocr dataset","document ai indic",
 "image text dataset indian","deepfake detection dataset",
 # --- speech / audio ---
 "indic speech dataset asr","speech translation indic","audio abuse detection dataset",
 "low resource speech corpus","code-mixed speech dataset",
 # --- low-resource languages ---
 "kashmiri nlp","sindhi corpus nlp","pashto nlp dataset","tibetan nlp corpus",
 "uyghur corpus nlp","bodo language dataset","santali language nlp","manipuri nlp dataset",
 "maithili nlp","dogri language corpus",
 # --- instruction / alignment / general ---
 "indic instruction tuning dataset","indian language llm dataset","indic benchmark llm",
 # --- OSINT tooling ---
 "osint toolkit","social media analysis toolkit","open source intelligence framework",
]
seen=json.load(open(OUT)) if os.path.exists(OUT) else {}
print(f"starting from {len(seen)} known repos",flush=True)
for i,q in enumerate(QUERIES,1):
    for page in (1,2):
        try:
            u=("https://api.github.com/search/repositories?q="+urllib.parse.quote(q)+
               f"&sort=stars&order=desc&per_page=30&page={page}")
            r=api(u)
            for it in r.get("items",[]):
                k=it["full_name"]
                if k in seen: continue
                seen[k]={"stars":it["stargazers_count"],"desc":(it.get("description") or "")[:150],
                         "url":it["clone_url"],"size_kb":it.get("size",0),"q":q,
                         "lang":it.get("language"),"pushed":it.get("pushed_at","")[:10]}
        except Exception as e:
            print(f"ERR {q} p{page}: {str(e)[:70]}",flush=True); break
        time.sleep(2.2)
    if i%10==0: print(f"  [{i}/{len(QUERIES)}] {len(seen)} repos",flush=True)
json.dump(seen,open(OUT,"w"),indent=2)
print(f"\nTOTAL {len(seen)} distinct repos",flush=True)