Instructions to use SinhalaMMLU-SharedTask/team-sinbotics with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use SinhalaMMLU-SharedTask/team-sinbotics with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
File size: 1,817 Bytes
7a2335a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 | """Average the option probabilities of finished experiments (same model, same data) into a new experiment.
No model is run; the result is saved like any other experiment (dev: results row; test: predictions.zip).
python -m sinmmlu.ensemble --name ens_b1_b3_c1 --exps <exp_id> <exp_id> [<exp_id> ...]
The experiment id becomes <name>__<data tag>. All parts must use the same base model (the 8B limit counts each model
once, so prompts of one model can be combined freely).
"""
import argparse
import json
from .data import ROOT, load_jsonl, load_rows, path
from .evaluate import save_experiment
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--name", required=True)
ap.add_argument("--exps", nargs="+", required=True)
a = ap.parse_args()
cfgs = [json.loads((ROOT / "outputs" / e / "config.json").read_text(encoding="utf-8")) for e in a.exps]
data = {c["data"] for c in cfgs}
models = {c.get("model_cfg", {}).get("model") for c in cfgs}
assert len(data) == 1, f"experiments use different data: {data}"
assert len(models) == 1, f"experiments use different models: {models}"
data = data.pop()
rows = load_rows(data)
preds = [{p["uid"]: p["probs"] for p in load_jsonl(ROOT / "outputs" / e / "preds.jsonl")} for e in a.exps]
for e, pr in zip(a.exps, preds):
assert set(pr) == {r["uid"] for r in rows}, f"{e} does not cover {data}"
probs = [[sum(pr[r["uid"]][j] for pr in preds) / len(preds) for j in range(4)] for r in rows]
tag = "test" if data == "test" else path(data).stem
save_experiment(f"{a.name}__{tag}", data, rows, probs, {},
{"method": "ensemble", "parts": a.exps, "model_cfg": cfgs[0].get("model_cfg", {}),
"env": cfgs[0].get("env", {})})
if __name__ == "__main__":
main()
|