"""Honest evaluation: the changing-facts test and calibration. The two headline measurements this project is built around. The deterministic emulator answers grounded in memory, so these numbers are reproducible and honest: no model-size claims, just this memory system's behaviour. """ from __future__ import annotations import time from .confidence import effective_confidence from .memory import Memory # Question -> expected correct answer. BENCHMARK = [ ("the price of apple", "10"), ("what is an apple", "apple is a fruit"), ("what is python", "python is a programming language"), ("what is the capital of france", "paris"), ] def calibration_observations(m: Memory, questions): """Run questions, record (confidence, whether answer was correct).""" obs = [] for q, expected in questions: res = m.ask(q) correct = bool(expected) and expected.lower() in res["answer"].lower() obs.append({"confidence": res["confidence"], "correct": correct}) return obs def _ece(obs): confs = [float(o["confidence"]) for o in obs] accs = [1.0 if o["correct"] else 0.0 for o in obs] ece = 0.0 n = len(confs) edges = [i / 10 for i in range(11)] for i in range(10): idx = [c for c in range(n) if edges[i] <= confs[c] < edges[i + 1]] if idx: frac = len(idx) / n avg_conf = sum(confs[c] for c in idx) / len(idx) avg_acc = sum(accs[c] for c in idx) / len(idx) ece += frac * abs(avg_conf - avg_acc) return ece def full_report(m: Memory, k: int = 20) -> dict: obs = calibration_observations(m, BENCHMARK) return { "ece": round(_ece(obs), 3), "n": len(obs), "accuracy": round(sum(o["correct"] for o in obs) / max(1, len(obs)), 3), "observations": obs, } def calibration_report(m: Memory, k: int = 20) -> dict: obs = calibration_observations(m, BENCHMARK) return { "ece": round(_ece(obs), 3), "accuracy": round(sum(o["correct"] for o in obs) / max(1, len(obs)), 3), "n": len(obs), } def changing_facts_report(m: Memory) -> dict: """The signature test: fast facts must be forgotten, static facts kept. Steps: A. Plant fast and static facts. B. Advance time past 5 half-lives (10 days), then run an expiry sweep. C. Score stale-answer rate and retention of static facts. """ m.learn("price of apple is 10", source_type="user", confidence=0.9, volatility="fast") m.learn("apple is a fruit", source_type="user", confidence=0.9, volatility="static") m.learn("paris is a city", source_type="official", confidence=0.95, volatility="static") before = m.size() later = time.time() + 10 * 86400 # 5 half-lives for fast (2-day) volatility exp = effective_confidence(0.9, 2.0, later, time.time()) retracted = m.store.expiry_sweep(now=later) size_after = m.size() res = m.ask("the price of apple") stale = "10" in res["answer"] and "confident memory" not in res["answer"] and "sure" not in res["answer"] Paris = m.ask("what is paris").get("answer", "") retained = "paris" in Paris.lower() and "city" in Paris.lower() return { "fast_half_life_days": 2, "simulated_days": 10, "effective_confidence_after": round(exp, 4), "retracted_count": len(retracted), "stale_answer_rate": 1 if stale else 0, "retention": 1 if retained else 0, "final_answer": res["answer"], "sources": res["sources"], }