"""AUBIN-Learn ölçümü (CPU yeter): öz-öğrenme belleği + kayıtlı model tahminleri → kilitli Kev testleri. Protokol (dürüst): * Bellek = Kev decision-v7 TRAIN (+ kev_augment ek verisi). Test/dev/cal/transfer kümeleri belleğe ASLA girmez. * Model log-olasılıkları önceden ölçülmüş koşulardan (aynı madde sırası; y eşleşmesi doğrulanır). * Birleştirme ağırlığı (kaynak-başına, log-kayıpla), komşu sayısı ve sıcaklık YALNIZ dev'de (kev_dev; yoksa cal-300) seçilir; kev_test / kev_transfer_test bir kez raporlanır. * Ek: öğrenme eğrisi (bellek %0→%100), gecikme (learn/recall ms), çevrimiçi geri bildirim deneyi (ayrı protokol). python learn_eval.py --work /tmp/kev --pack model_lp.json.gz --embed bge,hash --out learn_report.json """ import argparse, gzip, json, math, os, random, sys, time import numpy as np HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) import kevdata from kev_llm import items from aubin.learn import DecisionMemory, hash_embed, fuse, sig, vote SUITES = ("kev_dev", "kev_test", "kev_transfer_dev", "kev_transfer_test") WGRID = [0.0, 0.1, 0.2, 0.35, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0] def bge_embed_factory(name="BAAI/bge-small-en-v1.5"): try: from sentence_transformers import SentenceTransformer except ImportError: import subprocess subprocess.run([sys.executable, "-m", "pip", "-q", "install", "sentence-transformers"], check=False) from sentence_transformers import SentenceTransformer m = SentenceTransformer(name, device="cpu"); m.max_seq_length = int(os.environ.get("AUBIN_EMB_LEN", 128)) return lambda texts: m.encode(list(texts), batch_size=64, normalize_embeddings=True, convert_to_numpy=True).astype(np.float32) def text(it): return f"{it['state']} || {it['q']}" def acc(rows): return float(np.mean([int(np.argmax(lp)) == y for lp, y in rows])) if rows else float("nan") def nll(rows): return float(np.mean([-lp[y] for lp, y in rows])) if rows else float("nan") def neighbours(mem, its, V, K=64): out = [] for it, v in zip(its, V): s = sig(it["src"], it["keys"]); mem._freeze(s) M = mem.mats.get(s) if M is None or not len(M): out.append(None); continue sims = M @ v; kk = min(K, len(sims)); top = np.argpartition(-sims, kk - 1)[:kk] top = top[np.argsort(-sims[top])] out.append((sims[top].astype(np.float32), np.asarray([mem.labels[s][i] for i in top]))) return out def pmem(nb, keys, k, tau): if nb is None: return None sims, labs = nb[0][:k], nb[1][:k] return vote(list(labs), np.exp((sims - sims.max()) / tau), keys) def combine(lps, ys, nbs, srcs, kss, k, tau, wsrc, T): return [(fuse(lp, pmem(nb, ks, k, tau), wsrc.get(s, wsrc.get("*", 0.0)), T), y) for lp, y, nb, s, ks in zip(lps, ys, nbs, srcs, kss)] def tune(dev, T): """Global (k, τ, w) dev log-kaybıyla; sonra kaynak-başına w (≥30 madde) yine log-kayıpla (doğruluktan daha az gürültülü).""" lps, ys, nbs, srcs, ns = dev best = None for k in (8, 16, 32, 64): for tau in (0.02, 0.05, 0.1, 0.2): for w in WGRID: v = nll(combine(lps, ys, nbs, srcs, ns, k, tau, {"*": w}, T)) if best is None or v < best[0] - 1e-9: best = (v, k, tau, w) _, k, tau, wg = best wsrc = {"*": wg} for s in sorted(set(srcs)): idx = [i for i, x in enumerate(srcs) if x == s] if len(idx) < 30: continue sub = [lps[i] for i in idx], [ys[i] for i in idx], [nbs[i] for i in idx], [srcs[i] for i in idx], [ns[i] for i in idx] wsrc[s] = min(WGRID, key=lambda w: (nll(combine(*sub, k, tau, {"*": w}, T)), abs(w - wg))) full = combine(lps, ys, nbs, srcs, ns, k, tau, wsrc, T) return {"k": k, "tau": tau, "w": wsrc, "dev_nll_global": round(best[0], 4), "dev_acc_fused": round(acc(full), 4)} def by_source(rows, srcs): d = {} for (lp, y), s in zip(rows, srcs): d.setdefault(s, []).append(int(np.argmax(lp)) == y) return {s: [round(float(np.mean(v)), 4), len(v)] for s, v in sorted(d.items())} def main(): ap = argparse.ArgumentParser() ap.add_argument("--work", default="/tmp/kev"); ap.add_argument("--pack", required=True) ap.add_argument("--embed", default="bge,hash"); ap.add_argument("--out", default="learn_report.json") a = ap.parse_args() K = kevdata.load(a.work) tr = items(K["kev_train"]) ev = {s: items(K[s]) for s in SUITES} cal = items(K["kev_cal"]); random.Random(0).shuffle(cal); ev["cal300"] = cal[:300] pack = json.load(gzip.open(a.pack, "rt", encoding="utf-8")) is_extra = np.array([x["key"].startswith("kev_extra/") for x in tr]) R = {"protocol": __doc__.split("Protokol (dürüst):")[1].split("python learn_eval")[0].strip(), "memory": {"kev_train_items": int((~is_extra).sum()), "extra_items": int(is_extra.sum())}, "reference": {"Kev-9B kev_test": 0.874, "Kev-27B kev_test": 0.870, "Kev-27B kev_transfer_test": 0.896}, "embeds": {}} for emb in a.embed.split(","): t0 = time.time() fn = hash_embed if emb == "hash" else bge_embed_factory({"bge": "BAAI/bge-small-en-v1.5", "bge-base": "BAAI/bge-base-en-v1.5", "e5-base": "intfloat/e5-base-v2"}[emb]) VP = np.concatenate([fn([text(x) for x in tr[i:i + 4096]]) for i in range(0, len(tr), 4096)]) VE = {s: fn([text(x) for x in ev[s]]) for s in ev} print(emb, "gömme bitti", VP.shape, round(time.time() - t0), "s", flush=True) if emb != "hash": # sonraki koşular için gömmeleri sakla (float16) np.savez_compressed(os.path.join(os.path.dirname(os.path.abspath(a.out)), f"emb_{emb}.npz"), VP=VP.astype(np.float16), **{"VE_" + s: VE[s].astype(np.float16) for s in VE}) out = {} for mem_name, mask in (("kev+extra", np.ones(len(tr), bool)), ("kev_only", ~is_extra)): if mem_name == "kev+extra" and not is_extra.any(): continue idx = np.where(mask)[0] mem = DecisionMemory(fn) mem.learn_vectors(VP[idx], [tr[i]["src"] for i in idx], [tr[i]["keys"] for i in idx], [tr[i]["y"] for i in idx]) NB = {s: neighbours(mem, ev[s], VE[s]) for s in ev} res = {"embed_seconds": round(time.time() - t0, 1), "memory_items": int(len(idx)), "runs": {}} it0 = ev["kev_test"][0] ms_l = [mem.learn(text(it0), "latency_probe", it0["keys"], it0["y"]) for _ in range(20)] t1 = time.perf_counter(); [mem.recall(text(x), x["src"], x["keys"]) for x in ev["kev_test"][:100]] res["latency_ms"] = {"learn_median": round(float(np.median(ms_l)), 3), "recall_mean_incl_embed": round((time.perf_counter() - t1) * 1e3 / 100, 3)} for rn, rp in pack.items(): T = rp.get("T", 1.0); have = [s for s in ev if s in rp["suites"]] def prep(s, nbs=None): its = ev[s]; L = rp["suites"][s] assert len(L["y"]) == len(its) and all(int(y) == it["y"] for y, it in zip(L["y"], its)), f"{rn}/{s} sıra uyuşmuyor" return [np.asarray(x) for x in L["lp"]], L["y"], (nbs or NB[s]), [it["src"] for it in its], [it["keys"] for it in its] devname = "kev_dev" if "kev_dev" in have else ("cal300" if "cal300" in have else None) if devname is None: continue prm = tune(prep(devname), T) rr = {"dev_used": devname, "params": prm} for s in ("kev_test", "kev_transfer_test", "kev_dev", "kev_transfer_dev"): if s not in have: continue P = prep(s) base = combine(*P, prm["k"], prm["tau"], {"*": 0.0}, T) fu = combine(*P, prm["k"], prm["tau"], prm["w"], T) memonly = [((np.log(pm + 1e-9) if pm is not None else lp), y) for lp, y, pm in zip(P[0], P[1], [pmem(nb, ks, prm["k"], prm["tau"]) for nb, ks in zip(P[2], P[4])])] rr[s] = {"model": round(acc(base), 4), "memory_only": round(acc(memonly), 4), "fused": round(acc(fu), 4), "model_nll": round(nll(base), 4), "fused_nll": round(nll(fu), 4), "n": len(base), "by_source_model": by_source(base, P[3]), "by_source_fused": by_source(fu, P[3])} if "kev_test" in have and mem_name == "kev+extra": # öğrenme eğrisi: bellek büyüdükçe test doğruluğu curve = {} rng = np.random.default_rng(1); u = rng.random(len(idx)) for f in (0.0, 0.05, 0.2, 0.5, 1.0): sel = idx[u < f]; sub = DecisionMemory(fn) if len(sel): sub.learn_vectors(VP[sel], [tr[i]["src"] for i in sel], [tr[i]["keys"] for i in sel], [tr[i]["y"] for i in sel]) P = prep("kev_test", neighbours(sub, ev["kev_test"], VE["kev_test"])) curve[str(f)] = {"memory_items": int(len(sel)), "kev_test_fused": round(acc(combine(*P, prm["k"], prm["tau"], prm["w"], T)), 4)} rr["learning_curve"] = curve if "kev_test" in have: # çevrimiçi geri bildirim: test akışında her cevaptan sonra doğru etiket belleğe on = DecisionMemory(fn) on.learn_vectors(VP[idx], [tr[i]["src"] for i in idx], [tr[i]["keys"] for i in idx], [tr[i]["y"] for i in idx]) hits = [] for j, it in enumerate(ev["kev_test"]): nb = neighbours(on, [it], VE["kev_test"][j:j + 1])[0] lp = fuse(np.asarray(rp["suites"]["kev_test"]["lp"][j]), pmem(nb, it["keys"], prm["k"], prm["tau"]), prm["w"].get(it["src"], prm["w"]["*"]), T) hits.append(int(np.argmax(lp)) == it["y"]) on.learn_vectors(VE["kev_test"][j:j + 1], [it["src"]], [it["keys"]], [it["y"]]) rr["online_feedback_kev_test"] = {"note": "ayrı protokol: her cevaptan sonra doğru etiket belleğe yazılır (statik test değil)", "accuracy": round(float(np.mean(hits)), 4), "second_half": round(float(np.mean(hits[len(hits) // 2:])), 4)} res["runs"][rn] = rr print(emb, mem_name, rn, "TEST fused", {s: rr[s]["fused"] for s in ("kev_test", "kev_transfer_test") if s in rr}, "model", {s: rr[s]["model"] for s in ("kev_test", "kev_transfer_test") if s in rr}, flush=True) out[mem_name] = res R["embeds"][emb] = out json.dump(R, open(a.out, "w"), indent=1) print("BITTI", a.out, flush=True) if __name__ == "__main__": main()