File size: 3,551 Bytes
fbb4123 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 | """Honest evaluation: the changing-facts test and calibration.
The two headline measurements this project is built around. The deterministic
emulator answers grounded in memory, so these numbers are reproducible and
honest: no model-size claims, just this memory system's behaviour.
"""
from __future__ import annotations
import time
from .confidence import effective_confidence
from .memory import Memory
# Question -> expected correct answer.
BENCHMARK = [
("the price of apple", "10"),
("what is an apple", "apple is a fruit"),
("what is python", "python is a programming language"),
("what is the capital of france", "paris"),
]
def calibration_observations(m: Memory, questions):
"""Run questions, record (confidence, whether answer was correct)."""
obs = []
for q, expected in questions:
res = m.ask(q)
correct = bool(expected) and expected.lower() in res["answer"].lower()
obs.append({"confidence": res["confidence"], "correct": correct})
return obs
def _ece(obs):
confs = [float(o["confidence"]) for o in obs]
accs = [1.0 if o["correct"] else 0.0 for o in obs]
ece = 0.0
n = len(confs)
edges = [i / 10 for i in range(11)]
for i in range(10):
idx = [c for c in range(n) if edges[i] <= confs[c] < edges[i + 1]]
if idx:
frac = len(idx) / n
avg_conf = sum(confs[c] for c in idx) / len(idx)
avg_acc = sum(accs[c] for c in idx) / len(idx)
ece += frac * abs(avg_conf - avg_acc)
return ece
def full_report(m: Memory, k: int = 20) -> dict:
obs = calibration_observations(m, BENCHMARK)
return {
"ece": round(_ece(obs), 3),
"n": len(obs),
"accuracy": round(sum(o["correct"] for o in obs) / max(1, len(obs)), 3),
"observations": obs,
}
def calibration_report(m: Memory, k: int = 20) -> dict:
obs = calibration_observations(m, BENCHMARK)
return {
"ece": round(_ece(obs), 3),
"accuracy": round(sum(o["correct"] for o in obs) / max(1, len(obs)), 3),
"n": len(obs),
}
def changing_facts_report(m: Memory) -> dict:
"""The signature test: fast facts must be forgotten, static facts kept.
Steps:
A. Plant fast and static facts.
B. Advance time past 5 half-lives (10 days), then run an expiry sweep.
C. Score stale-answer rate and retention of static facts.
"""
m.learn("price of apple is 10", source_type="user", confidence=0.9, volatility="fast")
m.learn("apple is a fruit", source_type="user", confidence=0.9, volatility="static")
m.learn("paris is a city", source_type="official", confidence=0.95, volatility="static")
before = m.size()
later = time.time() + 10 * 86400 # 5 half-lives for fast (2-day) volatility
exp = effective_confidence(0.9, 2.0, later, time.time())
retracted = m.store.expiry_sweep(now=later)
size_after = m.size()
res = m.ask("the price of apple")
stale = "10" in res["answer"] and "confident memory" not in res["answer"] and "sure" not in res["answer"]
Paris = m.ask("what is paris").get("answer", "")
retained = "paris" in Paris.lower() and "city" in Paris.lower()
return {
"fast_half_life_days": 2,
"simulated_days": 10,
"effective_confidence_after": round(exp, 4),
"retracted_count": len(retracted),
"stale_answer_rate": 1 if stale else 0,
"retention": 1 if retained else 0,
"final_answer": res["answer"],
"sources": res["sources"],
}
|