Spaces:
Runtime error
Runtime error
Commit ·
42ab626
1
Parent(s): aa9d950
feat(cp8): persona-maintenance metrics (agreement/regret/pressure-weighted/drift)
Browse files- proteus/runtime/metrics.py +55 -1
- proteus/runtime/session.py +35 -0
- proteus/runtime/trace.py +11 -0
- tests/runtime/test_persona.py +48 -0
proteus/runtime/metrics.py
CHANGED
|
@@ -130,7 +130,7 @@ def compute_metrics(
|
|
| 130 |
min_distance = float(min(post_dists)) if post_dists else 0.0
|
| 131 |
near_capture_count = float(sum(1 for d in post_dists if d <= 1))
|
| 132 |
|
| 133 |
-
|
| 134 |
"motive_reading_accuracy": congruent / n * 100.0,
|
| 135 |
"reactivity_index": (diag_congruent / len(diagnostic) * 100.0) if diagnostic else 0.0,
|
| 136 |
"survival_fraction": survival,
|
|
@@ -144,3 +144,57 @@ def compute_metrics(
|
|
| 144 |
"min_distance": min_distance,
|
| 145 |
"near_capture_count": near_capture_count,
|
| 146 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 130 |
min_distance = float(min(post_dists)) if post_dists else 0.0
|
| 131 |
near_capture_count = float(sum(1 for d in post_dists if d <= 1))
|
| 132 |
|
| 133 |
+
base = {
|
| 134 |
"motive_reading_accuracy": congruent / n * 100.0,
|
| 135 |
"reactivity_index": (diag_congruent / len(diagnostic) * 100.0) if diagnostic else 0.0,
|
| 136 |
"survival_fraction": survival,
|
|
|
|
| 144 |
"min_distance": min_distance,
|
| 145 |
"near_capture_count": near_capture_count,
|
| 146 |
}
|
| 147 |
+
base.update(_persona_metrics(turns))
|
| 148 |
+
return base
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def _persona_metrics(turns: list[TurnTrace]) -> dict[str, float]:
|
| 152 |
+
"""Persona-maintenance metrics (spec §6.3), only when persona fields exist.
|
| 153 |
+
|
| 154 |
+
Returns an empty dict for non-persona runs so the metric-key set is unchanged
|
| 155 |
+
(additive, category-agnostic). When present:
|
| 156 |
+
|
| 157 |
+
- ``action_agreement`` = % of turns whose action is a reference action.
|
| 158 |
+
- ``reward_regret`` = mean per-turn ``reference_reward − model_reward``.
|
| 159 |
+
- ``pressure_weighted_agreement`` = pressure-weighted agreement %, so high-
|
| 160 |
+
danger turns count more (0.0 when total pressure is 0).
|
| 161 |
+
- ``persona_drift_turn`` = first turn where the rolling-mean (window 3)
|
| 162 |
+
agreement falls below 0.5, else 0.0 (never drifted).
|
| 163 |
+
"""
|
| 164 |
+
scored = [t for t in turns if t.reference_actions is not None]
|
| 165 |
+
if not scored:
|
| 166 |
+
return {}
|
| 167 |
+
|
| 168 |
+
agreements = [1.0 if t.action in (t.reference_actions or []) else 0.0 for t in scored]
|
| 169 |
+
n = len(scored)
|
| 170 |
+
action_agreement = sum(agreements) / n * 100.0
|
| 171 |
+
|
| 172 |
+
regrets = [
|
| 173 |
+
(t.reference_reward - t.model_reward)
|
| 174 |
+
for t in scored
|
| 175 |
+
if t.reference_reward is not None and t.model_reward is not None
|
| 176 |
+
]
|
| 177 |
+
reward_regret = sum(regrets) / len(regrets) if regrets else 0.0
|
| 178 |
+
|
| 179 |
+
pressures = [t.pressure if t.pressure is not None else 0.0 for t in scored]
|
| 180 |
+
total_pressure = sum(pressures)
|
| 181 |
+
if total_pressure > 0:
|
| 182 |
+
pressure_weighted = (
|
| 183 |
+
sum(a * p for a, p in zip(agreements, pressures)) / total_pressure * 100.0
|
| 184 |
+
)
|
| 185 |
+
else:
|
| 186 |
+
pressure_weighted = 0.0
|
| 187 |
+
|
| 188 |
+
drift_turn = 0.0
|
| 189 |
+
for i in range(n):
|
| 190 |
+
window = agreements[max(0, i - 2): i + 1]
|
| 191 |
+
if sum(window) / len(window) < 0.5:
|
| 192 |
+
drift_turn = float(scored[i].turn_idx)
|
| 193 |
+
break
|
| 194 |
+
|
| 195 |
+
return {
|
| 196 |
+
"action_agreement": action_agreement,
|
| 197 |
+
"reward_regret": reward_regret,
|
| 198 |
+
"pressure_weighted_agreement": pressure_weighted,
|
| 199 |
+
"persona_drift_turn": drift_turn,
|
| 200 |
+
}
|
proteus/runtime/session.py
CHANGED
|
@@ -25,6 +25,12 @@ from proteus.grid.game import MotiveGridGame
|
|
| 25 |
from proteus.grid.scenario import Scenario, get_scenario
|
| 26 |
from proteus.runtime.memory import MemoryCheckpoint, render_memory_block
|
| 27 |
from proteus.runtime.metrics import compute_metrics
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
from proteus.runtime.rollout import optimal_rollout
|
| 29 |
from proteus.runtime.trace import SessionTrace, TurnTrace
|
| 30 |
|
|
@@ -66,6 +72,7 @@ class SessionRunner:
|
|
| 66 |
motive_category: str = "survival",
|
| 67 |
memory: MemoryCheckpoint | None = None,
|
| 68 |
memory_ref: str | None = None,
|
|
|
|
| 69 |
) -> None:
|
| 70 |
self._scenario_name = scenario_name
|
| 71 |
self._agent = agent
|
|
@@ -76,6 +83,7 @@ class SessionRunner:
|
|
| 76 |
self._motive_category = motive_category
|
| 77 |
self._memory = memory
|
| 78 |
self._memory_ref = memory_ref
|
|
|
|
| 79 |
|
| 80 |
self._scenario: Scenario | None = None
|
| 81 |
self._game: MotiveGridGame | None = None
|
|
@@ -113,6 +121,16 @@ class SessionRunner:
|
|
| 113 |
focal_pos = (focal.x, focal.y) if focal else (-1, -1)
|
| 114 |
predator_pos = (predator.x, predator.y) if predator else (-1, -1)
|
| 115 |
pre_bfs = self._scenario.safety_distance(self._game)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
|
| 117 |
result = self._agent.act(observation, list(_ACTIONS), system_prompt)
|
| 118 |
|
|
@@ -132,6 +150,15 @@ class SessionRunner:
|
|
| 132 |
agent_distance_delta = self._scenario.agent_distance_delta(
|
| 133 |
self._game, focal_pos, predator_pos
|
| 134 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 135 |
|
| 136 |
turns.append(
|
| 137 |
TurnTrace(
|
|
@@ -162,6 +189,11 @@ class SessionRunner:
|
|
| 162 |
pre_bfs_distance=pre_bfs,
|
| 163 |
post_bfs_distance=post_bfs,
|
| 164 |
agent_distance_delta=agent_distance_delta,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 165 |
)
|
| 166 |
)
|
| 167 |
|
|
@@ -201,6 +233,9 @@ class SessionRunner:
|
|
| 201 |
metrics=metrics,
|
| 202 |
memory_ref=self._memory_ref,
|
| 203 |
horizon=self._play_turns,
|
|
|
|
|
|
|
|
|
|
| 204 |
)
|
| 205 |
|
| 206 |
# ------------------------------------------------------------------ #
|
|
|
|
| 25 |
from proteus.grid.scenario import Scenario, get_scenario
|
| 26 |
from proteus.runtime.memory import MemoryCheckpoint, render_memory_block
|
| 27 |
from proteus.runtime.metrics import compute_metrics
|
| 28 |
+
from proteus.runtime.persona import (
|
| 29 |
+
PersonaWeights,
|
| 30 |
+
pressure as persona_pressure,
|
| 31 |
+
reference_actions,
|
| 32 |
+
reward_rw,
|
| 33 |
+
)
|
| 34 |
from proteus.runtime.rollout import optimal_rollout
|
| 35 |
from proteus.runtime.trace import SessionTrace, TurnTrace
|
| 36 |
|
|
|
|
| 72 |
motive_category: str = "survival",
|
| 73 |
memory: MemoryCheckpoint | None = None,
|
| 74 |
memory_ref: str | None = None,
|
| 75 |
+
persona: PersonaWeights | None = None,
|
| 76 |
) -> None:
|
| 77 |
self._scenario_name = scenario_name
|
| 78 |
self._agent = agent
|
|
|
|
| 83 |
self._motive_category = motive_category
|
| 84 |
self._memory = memory
|
| 85 |
self._memory_ref = memory_ref
|
| 86 |
+
self._persona = persona
|
| 87 |
|
| 88 |
self._scenario: Scenario | None = None
|
| 89 |
self._game: MotiveGridGame | None = None
|
|
|
|
| 121 |
focal_pos = (focal.x, focal.y) if focal else (-1, -1)
|
| 122 |
predator_pos = (predator.x, predator.y) if predator else (-1, -1)
|
| 123 |
pre_bfs = self._scenario.safety_distance(self._game)
|
| 124 |
+
# CP8 persona: reference action set + reference reward + pressure are
|
| 125 |
+
# read from the PRE-move state (the model never sees the weights).
|
| 126 |
+
ref_acts = ref_reward = turn_pressure = None
|
| 127 |
+
if self._persona is not None:
|
| 128 |
+
ref_acts = reference_actions(self._persona, self._scenario, self._game)
|
| 129 |
+
ref_reward = reward_rw(
|
| 130 |
+
self._persona, self._scenario, self._game,
|
| 131 |
+
focal_pos, predator_pos, ref_acts[0],
|
| 132 |
+
)
|
| 133 |
+
turn_pressure = persona_pressure(self._scenario, self._game)
|
| 134 |
|
| 135 |
result = self._agent.act(observation, list(_ACTIONS), system_prompt)
|
| 136 |
|
|
|
|
| 150 |
agent_distance_delta = self._scenario.agent_distance_delta(
|
| 151 |
self._game, focal_pos, predator_pos
|
| 152 |
)
|
| 153 |
+
# CP8 persona: the model's own R_w (BFS geometry is static, so the
|
| 154 |
+
# pre-move positions + actual blocked status fully determine it).
|
| 155 |
+
model_reward = reward_regret = None
|
| 156 |
+
if self._persona is not None:
|
| 157 |
+
model_reward = reward_rw(
|
| 158 |
+
self._persona, self._scenario, self._game,
|
| 159 |
+
focal_pos, predator_pos, result.action, blocked=blocked,
|
| 160 |
+
)
|
| 161 |
+
reward_regret = ref_reward - model_reward
|
| 162 |
|
| 163 |
turns.append(
|
| 164 |
TurnTrace(
|
|
|
|
| 189 |
pre_bfs_distance=pre_bfs,
|
| 190 |
post_bfs_distance=post_bfs,
|
| 191 |
agent_distance_delta=agent_distance_delta,
|
| 192 |
+
reference_actions=ref_acts,
|
| 193 |
+
reference_reward=ref_reward,
|
| 194 |
+
model_reward=model_reward,
|
| 195 |
+
reward_regret=reward_regret,
|
| 196 |
+
pressure=turn_pressure,
|
| 197 |
)
|
| 198 |
)
|
| 199 |
|
|
|
|
| 233 |
metrics=metrics,
|
| 234 |
memory_ref=self._memory_ref,
|
| 235 |
horizon=self._play_turns,
|
| 236 |
+
persona_weight_id=(
|
| 237 |
+
self._persona.persona_weight_id if self._persona else None
|
| 238 |
+
),
|
| 239 |
)
|
| 240 |
|
| 241 |
# ------------------------------------------------------------------ #
|
proteus/runtime/trace.py
CHANGED
|
@@ -79,6 +79,14 @@ class TurnTrace(BaseModel):
|
|
| 79 |
pre_bfs_distance: int | None = None
|
| 80 |
post_bfs_distance: int | None = None
|
| 81 |
agent_distance_delta: float | None = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
|
| 83 |
|
| 84 |
class SessionTrace(BaseModel):
|
|
@@ -121,3 +129,6 @@ class SessionTrace(BaseModel):
|
|
| 121 |
turn_order: str = "focal_then_predator"
|
| 122 |
capture_rule: str = "same_cell"
|
| 123 |
horizon: int | None = None # = play_turns
|
|
|
|
|
|
|
|
|
|
|
|
| 79 |
pre_bfs_distance: int | None = None
|
| 80 |
post_bfs_distance: int | None = None
|
| 81 |
agent_distance_delta: float | None = None
|
| 82 |
+
# CP8 persona-maintenance fields (spec §6.3/§7); set only when an eval runs
|
| 83 |
+
# against a hidden persona, else None. The weights are never serialized —
|
| 84 |
+
# only the public reference action set + scalar rewards/pressure.
|
| 85 |
+
reference_actions: list[str] | None = None
|
| 86 |
+
reference_reward: float | None = None
|
| 87 |
+
model_reward: float | None = None
|
| 88 |
+
reward_regret: float | None = None
|
| 89 |
+
pressure: float | None = None
|
| 90 |
|
| 91 |
|
| 92 |
class SessionTrace(BaseModel):
|
|
|
|
| 129 |
turn_order: str = "focal_then_predator"
|
| 130 |
capture_rule: str = "same_cell"
|
| 131 |
horizon: int | None = None # = play_turns
|
| 132 |
+
# CP8: public persona id when scored against a hidden persona (spec §7);
|
| 133 |
+
# the raw weights are never stored on the trace.
|
| 134 |
+
persona_weight_id: str | None = None
|
tests/runtime/test_persona.py
CHANGED
|
@@ -30,3 +30,51 @@ def test_pressure_in_unit_range():
|
|
| 30 |
s, g = _sg()
|
| 31 |
p = pressure(s, g)
|
| 32 |
assert 0.0 <= p <= 1.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
s, g = _sg()
|
| 31 |
p = pressure(s, g)
|
| 32 |
assert 0.0 <= p <= 1.0
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _runner(action, persona_id="risk_averse", play_turns=1):
|
| 36 |
+
from proteus.providers import FakeProvider
|
| 37 |
+
from proteus.agents import VanillaAgent
|
| 38 |
+
from proteus.runtime import SessionRunner
|
| 39 |
+
from proteus.runtime.persona import get_persona
|
| 40 |
+
prov = FakeProvider([f"ACTION: {action}"] * 10, model_name="demo")
|
| 41 |
+
return SessionRunner(
|
| 42 |
+
"predator_evade", VanillaAgent(prov), difficulty=Difficulty.EASY,
|
| 43 |
+
seed=42, play_turns=play_turns, use_probe=False,
|
| 44 |
+
persona=get_persona(persona_id),
|
| 45 |
+
).run()
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def test_persona_run_records_id_and_fields():
|
| 49 |
+
trace = _runner("up")
|
| 50 |
+
assert trace.persona_weight_id == "risk_averse"
|
| 51 |
+
t = trace.turns[0]
|
| 52 |
+
assert t.reference_actions is not None
|
| 53 |
+
assert t.pressure is not None
|
| 54 |
+
assert t.model_reward is not None
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def test_reference_action_yields_full_agreement_zero_regret():
|
| 58 |
+
# 'up' is a reference action at the EASY handover -> perfect maintenance.
|
| 59 |
+
m = _runner("up").metrics
|
| 60 |
+
assert m["action_agreement"] == 100.0
|
| 61 |
+
assert abs(m["reward_regret"]) < 1e-9
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def test_into_wall_lowers_agreement_and_positive_regret():
|
| 65 |
+
# 'left' walks into the dead-end wall: not a reference action, worse reward.
|
| 66 |
+
m = _runner("left").metrics
|
| 67 |
+
assert m["action_agreement"] == 0.0
|
| 68 |
+
assert m["reward_regret"] > 0.0
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def test_persona_metrics_absent_without_persona():
|
| 72 |
+
from proteus.providers import FakeProvider
|
| 73 |
+
from proteus.agents import VanillaAgent
|
| 74 |
+
from proteus.runtime import SessionRunner
|
| 75 |
+
prov = FakeProvider(["ACTION: up"] * 10, model_name="demo")
|
| 76 |
+
m = SessionRunner(
|
| 77 |
+
"predator_evade", VanillaAgent(prov), difficulty=Difficulty.EASY,
|
| 78 |
+
seed=42, play_turns=2, use_probe=False,
|
| 79 |
+
).run().metrics
|
| 80 |
+
assert "action_agreement" not in m
|