File size: 3,040 Bytes
5daf54a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | """AUBIN ansambl + kalibrasyon (GPU'suz): kev_llm.py çıktılarındaki soru-başı log-olasılıkları birleştirir.
python ensemble.py out.json run1.json run2.json [--tag runs] [--weights 1,1]
Birleştirme: ağırlıklı log-olasılık ortalaması; sıcaklık YALNIZ calibration (decision-v7 cal) üzerinde seçilir.
"""
import argparse, json, math
import numpy as np
def softmax(x):
x = np.asarray(x, dtype=np.float64); x = x - x.max(); e = np.exp(x); return e / e.sum()
def metrics(rows, T):
by, allr = {}, []
for src, y, lp in rows:
p = softmax(np.asarray(lp) / T); oh = np.zeros(len(p)); oh[y] = 1
r = (int(p.argmax()) == y, float(((p - oh) ** 2).sum()), float(p.max()), -math.log(max(p[y], 1e-12)))
allr.append(r); by.setdefault(src, []).append(r)
def m(v):
ece = 0.0
for j in range(15):
bb = [x for x in v if j / 15 < x[2] <= (j + 1) / 15]
if bb:
ece += len(bb) / len(v) * abs(np.mean([x[2] for x in bb]) - np.mean([x[0] for x in bb]))
return {"n": len(v), "accuracy": round(float(np.mean([x[0] for x in v])), 4), "brier": round(float(np.mean([x[1] for x in v])), 4),
"nll": round(float(np.mean([x[3] for x in v])), 4), "ece15": round(float(ece), 4)}
return {"all": m(allr), "by_source": {k: m(v) for k, v in sorted(by.items())}}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("out"); ap.add_argument("runs", nargs="+"); ap.add_argument("--tag", default="runs")
ap.add_argument("--weights", default="")
a = ap.parse_args()
R = [json.load(open(f)) for f in a.runs]
w = [float(x) for x in a.weights.split(",")] if a.weights else [1.0] * len(R)
w = np.array(w) / sum(w)
cal = [r.get(a.tag + "_cal_items") for r in R]
out = {"members": [{"file": f, "model": r.get("model"), "lora": r.get("lora")} for f, r in zip(a.runs, R)], "weights": w.tolist()}
T = 1.0
if all(cal):
comb = [(None, c[0]["y"], sum(wi * np.asarray(ci[k]["lp"]) for wi, ci in zip(w, cal))) for k, c in enumerate(zip(*cal))]
T = min(np.concatenate([np.arange(0.3, 4.0, 0.05), np.arange(4.0, 20.01, 0.25)]), key=lambda t: metrics([(0, y, lp) for _, y, lp in comb], t)["all"]["nll"])
out["temperature"] = round(float(T), 2)
suites = set.intersection(*[set(r[a.tag].keys()) if a.tag in r else set(r["runs"].keys()) for r in R])
for s in sorted(suites):
its = [(r[a.tag] if a.tag in r else r["runs"])[s]["items"] for r in R]
assert all(len(x) == len(its[0]) for x in its), "üyelerin madde sayısı farklı"
rows = []
for k in range(len(its[0])):
ys = {x[k]["y"] for x in its}; assert len(ys) == 1, "madde sırası uyuşmuyor"
rows.append((its[0][k]["src"], its[0][k]["y"], sum(wi * np.asarray(x[k]["lp"]) for wi, x in zip(w, its))))
out[s] = metrics(rows, T)
print(s, json.dumps(out[s]["all"]))
json.dump(out, open(a.out, "w"), indent=1)
if __name__ == "__main__":
main()
|