Download humpback2/evaluate.py from gregrsea/HumpBack: direct link, hf CLI and curl.
- Browser
- Download file 3.55 kB
-
https://huggingface.co/gregrsea/HumpBack/resolve/main/humpback2/evaluate.py
- Command line
-
hf download hf://gregrsea/HumpBack/humpback2/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/gregrsea/HumpBack/resolve/main/humpback2/evaluate.py
3.55 kB
| """Honest evaluation: the changing-facts test and calibration. | |
| The two headline measurements this project is built around. The deterministic | |
| emulator answers grounded in memory, so these numbers are reproducible and | |
| honest: no model-size claims, just this memory system's behaviour. | |
| """ | |
| from __future__ import annotations | |
| import time | |
| from .confidence import effective_confidence | |
| from .memory import Memory | |
| # Question -> expected correct answer. | |
| BENCHMARK = [ | |
| ("the price of apple", "10"), | |
| ("what is an apple", "apple is a fruit"), | |
| ("what is python", "python is a programming language"), | |
| ("what is the capital of france", "paris"), | |
| ] | |
| def calibration_observations(m: Memory, questions): | |
| """Run questions, record (confidence, whether answer was correct).""" | |
| obs = [] | |
| for q, expected in questions: | |
| res = m.ask(q) | |
| correct = bool(expected) and expected.lower() in res["answer"].lower() | |
| obs.append({"confidence": res["confidence"], "correct": correct}) | |
| return obs | |
| def _ece(obs): | |
| confs = [float(o["confidence"]) for o in obs] | |
| accs = [1.0 if o["correct"] else 0.0 for o in obs] | |
| ece = 0.0 | |
| n = len(confs) | |
| edges = [i / 10 for i in range(11)] | |
| for i in range(10): | |
| idx = [c for c in range(n) if edges[i] <= confs[c] < edges[i + 1]] | |
| if idx: | |
| frac = len(idx) / n | |
| avg_conf = sum(confs[c] for c in idx) / len(idx) | |
| avg_acc = sum(accs[c] for c in idx) / len(idx) | |
| ece += frac * abs(avg_conf - avg_acc) | |
| return ece | |
| def full_report(m: Memory, k: int = 20) -> dict: | |
| obs = calibration_observations(m, BENCHMARK) | |
| return { | |
| "ece": round(_ece(obs), 3), | |
| "n": len(obs), | |
| "accuracy": round(sum(o["correct"] for o in obs) / max(1, len(obs)), 3), | |
| "observations": obs, | |
| } | |
| def calibration_report(m: Memory, k: int = 20) -> dict: | |
| obs = calibration_observations(m, BENCHMARK) | |
| return { | |
| "ece": round(_ece(obs), 3), | |
| "accuracy": round(sum(o["correct"] for o in obs) / max(1, len(obs)), 3), | |
| "n": len(obs), | |
| } | |
| def changing_facts_report(m: Memory) -> dict: | |
| """The signature test: fast facts must be forgotten, static facts kept. | |
| Steps: | |
| A. Plant fast and static facts. | |
| B. Advance time past 5 half-lives (10 days), then run an expiry sweep. | |
| C. Score stale-answer rate and retention of static facts. | |
| """ | |
| m.learn("price of apple is 10", source_type="user", confidence=0.9, volatility="fast") | |
| m.learn("apple is a fruit", source_type="user", confidence=0.9, volatility="static") | |
| m.learn("paris is a city", source_type="official", confidence=0.95, volatility="static") | |
| before = m.size() | |
| later = time.time() + 10 * 86400 # 5 half-lives for fast (2-day) volatility | |
| exp = effective_confidence(0.9, 2.0, later, time.time()) | |
| retracted = m.store.expiry_sweep(now=later) | |
| size_after = m.size() | |
| res = m.ask("the price of apple") | |
| stale = "10" in res["answer"] and "confident memory" not in res["answer"] and "sure" not in res["answer"] | |
| Paris = m.ask("what is paris").get("answer", "") | |
| retained = "paris" in Paris.lower() and "city" in Paris.lower() | |
| return { | |
| "fast_half_life_days": 2, | |
| "simulated_days": 10, | |
| "effective_confidence_after": round(exp, 4), | |
| "retracted_count": len(retracted), | |
| "stale_answer_rate": 1 if stale else 0, | |
| "retention": 1 if retained else 0, | |
| "final_answer": res["answer"], | |
| "sources": res["sources"], | |
| } | |