File size: 1,946 Bytes
9c90564 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | """Ek tur sonrasi olcek taramasi — esli bootstrap guven araliklari.
Soru-bazli vektorler ayni sorular uzerinde oldugu icin fark ESLI hesaplanir:
her yeniden orneklemede ayni soru indeksleri iki kosula da uygulanir. Bagimsiz
bootstrap'a gore araliklar daha dar ve dogru olur (soru zorlugu ortak varyans).
10.000 yeniden ornekleme, tohum 0 — OLCUM-PROTOKOLU ile ayni.
"""
import json, numpy as np
A = json.load(open("/ari/users/ytanis/olcek_ince_sft2.json")) # ek tur SONRASI
try:
B = json.load(open("/ari/users/ytanis/olcek_ince.json")) # ek tur ONCESI
except Exception:
B = {}
rng = np.random.default_rng(0)
N = 10000
def ci(x, y):
x = np.asarray(x, float); y = np.asarray(y, float); n = len(x)
assert n == len(y)
idx = rng.integers(0, n, size=(N, n))
d = (x[idx].mean(1) - y[idx].mean(1)) * 100
lo, hi = np.percentile(d, [2.5, 97.5])
return (x.mean() - y.mean()) * 100, lo, hi
def satir(ad, x, y):
f, lo, hi = ci(x, y)
anl = "ANLAMLI" if (lo > 0 or hi < 0) else " - "
print(" %-40s %+6.2f [%+6.2f, %+6.2f] %s" % (ad, f, lo, hi, anl))
print("EK TUR SONRASI — olcekler arasi (esli, n_sohbet=329, n_kiyas=652)")
print(" SOHBET")
for a, b in [("0.80", "0.00"), ("1.00", "0.00"), ("0.40", "0.00"), ("0.80", "0.40"), ("0.80", "1.00")]:
satir("sohbet %s vs %s" % (a, b), A[a]["sohbet_vec"], A[b]["sohbet_vec"])
print(" KIYAS")
for a, b in [("0.40", "0.00"), ("0.80", "0.00"), ("1.00", "0.00"), ("0.40", "0.80"), ("0.40", "1.00")]:
satir("kiyas %s vs %s" % (a, b), A[a]["mmlu_vec"], A[b]["mmlu_vec"])
if B:
print("\nEK TUR SONRASI vs ONCESI — ayni olcekte (esli)")
for o in ["0.40", "0.50", "1.00"]:
if o in B:
satir("sohbet %s: sonra vs once" % o, A[o]["sohbet_vec"], B[o]["sohbet_vec"])
satir("kiyas %s: sonra vs once" % o, A[o]["mmlu_vec"], B[o]["mmlu_vec"])
else:
print(" (%s once dosyasinda yok)" % o)
|