Download eval/sets_v1.py from sapinsapin/wikalid: direct link, hf CLI and curl.
- Browser
- Download file 2.07 kB
-
https://huggingface.co/sapinsapin/wikalid/resolve/main/eval/sets_v1.py
- Command line
-
hf download hf://sapinsapin/wikalid/eval/sets_v1.py
-
curl -L -o sets_v1.py https://huggingface.co/sapinsapin/wikalid/resolve/main/eval/sets_v1.py
2.07 kB
| """Eval-set loader for OurLID 1.0 (same split and scoring rules as the research repo's eval/sets.py, which produced every | |
| earlier number). Public sets ship in eval/data; private sets (news gold, crawl realism, PLD, Basaha, trust sets) are read | |
| from $OURLID_PRIVATE_EVAL and are never published.""" | |
| import hashlib, json, os | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| PRIV = os.environ.get("OURLID_PRIVATE_EVAL", "/workspace/ourlid/eval/private") | |
| CORE = {"crawl": f"{PRIV}/crawl_realism_eval.jsonl", "goldspan": f"{PRIV}/gold_spans.jsonl", "golddoc": f"{PRIV}/gold_docs.jsonl", | |
| "pld": f"{PRIV}/pld.jsonl", "wiki": f"{HERE}/data/wiki_test_v0.jsonl", "hard": f"{HERE}/data/phlid_hard_v0.jsonl", | |
| "basaha": f"{PRIV}/basaha2_eval.jsonl", "tato": f"{HERE}/data/tato_test_v0.jsonl"} | |
| TRUST = {"t_sib": f"{PRIV}/trust_sib.jsonl", "t_frag": f"{PRIV}/trust_frag.jsonl", "t_caps": f"{PRIV}/trust_caps.jsonl", | |
| "t_bikol": f"{PRIV}/trust_bikol.jsonl", "t_aklsgd": f"{PRIV}/trust_aklsgd.jsonl", "t_tshort": f"{PRIV}/trust_tshort.jsonl"} | |
| PUBLIC = {"wiki", "hard", "tato"} | |
| SETS = {**CORE, **TRUST} | |
| def available(): | |
| return {n: p for n, p in SETS.items() if os.path.exists(p)} | |
| def group(n, r): | |
| i = r["id"] | |
| if r.get("grp"): return r["grp"] | |
| if n == "goldspan": return i.rsplit("-", 1)[0] | |
| if n == "basaha": return "_".join(i.split("_")[:3]) | |
| return i | |
| def part_of(n, r): | |
| h = int(hashlib.md5(("sealed-v1|" + n + "|" + group(n, r)).encode()).hexdigest(), 16) | |
| return "sealed" if h % 2 == 0 else "dev" | |
| def rows(n, part="all"): | |
| R = [json.loads(l) for l in open(SETS[n], encoding="utf8")] | |
| if n == "crawl": R = [r for r in R if not r["uncertain"]] | |
| if n == "hard": R = [r for r in R if r["cat"] != "quoted_english"] | |
| return R if part == "all" else [r for r in R if part_of(n, r) == part] | |
| def ok(n, r, p): | |
| if n == "crawl" and r.get("ambig_frame") and p in ("tgl", "eng") and r["label"] in ("tgl", "eng"): return True | |
| if n == "basaha" and p in r.get("lenient", []): return True | |
| return p == r["label"] | |