File size: 3,551 Bytes
fbb4123
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
"""Honest evaluation: the changing-facts test and calibration.

The two headline measurements this project is built around. The deterministic
emulator answers grounded in memory, so these numbers are reproducible and
honest: no model-size claims, just this memory system's behaviour.
"""

from __future__ import annotations

import time

from .confidence import effective_confidence
from .memory import Memory

# Question -> expected correct answer.
BENCHMARK = [
    ("the price of apple", "10"),
    ("what is an apple", "apple is a fruit"),
    ("what is python", "python is a programming language"),
    ("what is the capital of france", "paris"),
]


def calibration_observations(m: Memory, questions):
    """Run questions, record (confidence, whether answer was correct)."""
    obs = []
    for q, expected in questions:
        res = m.ask(q)
        correct = bool(expected) and expected.lower() in res["answer"].lower()
        obs.append({"confidence": res["confidence"], "correct": correct})
    return obs


def _ece(obs):
    confs = [float(o["confidence"]) for o in obs]
    accs = [1.0 if o["correct"] else 0.0 for o in obs]
    ece = 0.0
    n = len(confs)
    edges = [i / 10 for i in range(11)]
    for i in range(10):
        idx = [c for c in range(n) if edges[i] <= confs[c] < edges[i + 1]]
        if idx:
            frac = len(idx) / n
            avg_conf = sum(confs[c] for c in idx) / len(idx)
            avg_acc = sum(accs[c] for c in idx) / len(idx)
            ece += frac * abs(avg_conf - avg_acc)
    return ece


def full_report(m: Memory, k: int = 20) -> dict:
    obs = calibration_observations(m, BENCHMARK)
    return {
        "ece": round(_ece(obs), 3),
        "n": len(obs),
        "accuracy": round(sum(o["correct"] for o in obs) / max(1, len(obs)), 3),
        "observations": obs,
    }


def calibration_report(m: Memory, k: int = 20) -> dict:
    obs = calibration_observations(m, BENCHMARK)
    return {
        "ece": round(_ece(obs), 3),
        "accuracy": round(sum(o["correct"] for o in obs) / max(1, len(obs)), 3),
        "n": len(obs),
    }


def changing_facts_report(m: Memory) -> dict:
    """The signature test: fast facts must be forgotten, static facts kept.

    Steps:
      A. Plant fast and static facts.
      B. Advance time past 5 half-lives (10 days), then run an expiry sweep.
      C. Score stale-answer rate and retention of static facts.
    """
    m.learn("price of apple is 10", source_type="user", confidence=0.9, volatility="fast")
    m.learn("apple is a fruit", source_type="user", confidence=0.9, volatility="static")
    m.learn("paris is a city", source_type="official", confidence=0.95, volatility="static")

    before = m.size()
    later = time.time() + 10 * 86400  # 5 half-lives for fast (2-day) volatility
    exp = effective_confidence(0.9, 2.0, later, time.time())
    retracted = m.store.expiry_sweep(now=later)
    size_after = m.size()

    res = m.ask("the price of apple")
    stale = "10" in res["answer"] and "confident memory" not in res["answer"] and "sure" not in res["answer"]
    Paris = m.ask("what is paris").get("answer", "")
    retained = "paris" in Paris.lower() and "city" in Paris.lower()

    return {
        "fast_half_life_days": 2,
        "simulated_days": 10,
        "effective_confidence_after": round(exp, 4),
        "retracted_count": len(retracted),
        "stale_answer_rate": 1 if stale else 0,
        "retention": 1 if retained else 0,
        "final_answer": res["answer"],
        "sources": res["sources"],
    }