| """Ek tur sonrasi olcek taramasi — esli bootstrap guven araliklari. |
| |
| Soru-bazli vektorler ayni sorular uzerinde oldugu icin fark ESLI hesaplanir: |
| her yeniden orneklemede ayni soru indeksleri iki kosula da uygulanir. Bagimsiz |
| bootstrap'a gore araliklar daha dar ve dogru olur (soru zorlugu ortak varyans). |
| 10.000 yeniden ornekleme, tohum 0 — OLCUM-PROTOKOLU ile ayni. |
| """ |
| import json, numpy as np |
|
|
| A = json.load(open("/ari/users/ytanis/olcek_ince_sft2.json")) |
| try: |
| B = json.load(open("/ari/users/ytanis/olcek_ince.json")) |
| except Exception: |
| B = {} |
| rng = np.random.default_rng(0) |
| N = 10000 |
|
|
| def ci(x, y): |
| x = np.asarray(x, float); y = np.asarray(y, float); n = len(x) |
| assert n == len(y) |
| idx = rng.integers(0, n, size=(N, n)) |
| d = (x[idx].mean(1) - y[idx].mean(1)) * 100 |
| lo, hi = np.percentile(d, [2.5, 97.5]) |
| return (x.mean() - y.mean()) * 100, lo, hi |
|
|
| def satir(ad, x, y): |
| f, lo, hi = ci(x, y) |
| anl = "ANLAMLI" if (lo > 0 or hi < 0) else " - " |
| print(" %-40s %+6.2f [%+6.2f, %+6.2f] %s" % (ad, f, lo, hi, anl)) |
|
|
| print("EK TUR SONRASI — olcekler arasi (esli, n_sohbet=329, n_kiyas=652)") |
| print(" SOHBET") |
| for a, b in [("0.80", "0.00"), ("1.00", "0.00"), ("0.40", "0.00"), ("0.80", "0.40"), ("0.80", "1.00")]: |
| satir("sohbet %s vs %s" % (a, b), A[a]["sohbet_vec"], A[b]["sohbet_vec"]) |
| print(" KIYAS") |
| for a, b in [("0.40", "0.00"), ("0.80", "0.00"), ("1.00", "0.00"), ("0.40", "0.80"), ("0.40", "1.00")]: |
| satir("kiyas %s vs %s" % (a, b), A[a]["mmlu_vec"], A[b]["mmlu_vec"]) |
|
|
| if B: |
| print("\nEK TUR SONRASI vs ONCESI — ayni olcekte (esli)") |
| for o in ["0.40", "0.50", "1.00"]: |
| if o in B: |
| satir("sohbet %s: sonra vs once" % o, A[o]["sohbet_vec"], B[o]["sohbet_vec"]) |
| satir("kiyas %s: sonra vs once" % o, A[o]["mmlu_vec"], B[o]["mmlu_vec"]) |
| else: |
| print(" (%s once dosyasinda yok)" % o) |
|
|
