irregular6612 commited on
Commit
42ab626
·
1 Parent(s): aa9d950

feat(cp8): persona-maintenance metrics (agreement/regret/pressure-weighted/drift)

Browse files
proteus/runtime/metrics.py CHANGED
@@ -130,7 +130,7 @@ def compute_metrics(
130
  min_distance = float(min(post_dists)) if post_dists else 0.0
131
  near_capture_count = float(sum(1 for d in post_dists if d <= 1))
132
 
133
- return {
134
  "motive_reading_accuracy": congruent / n * 100.0,
135
  "reactivity_index": (diag_congruent / len(diagnostic) * 100.0) if diagnostic else 0.0,
136
  "survival_fraction": survival,
@@ -144,3 +144,57 @@ def compute_metrics(
144
  "min_distance": min_distance,
145
  "near_capture_count": near_capture_count,
146
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
130
  min_distance = float(min(post_dists)) if post_dists else 0.0
131
  near_capture_count = float(sum(1 for d in post_dists if d <= 1))
132
 
133
+ base = {
134
  "motive_reading_accuracy": congruent / n * 100.0,
135
  "reactivity_index": (diag_congruent / len(diagnostic) * 100.0) if diagnostic else 0.0,
136
  "survival_fraction": survival,
 
144
  "min_distance": min_distance,
145
  "near_capture_count": near_capture_count,
146
  }
147
+ base.update(_persona_metrics(turns))
148
+ return base
149
+
150
+
151
+ def _persona_metrics(turns: list[TurnTrace]) -> dict[str, float]:
152
+ """Persona-maintenance metrics (spec §6.3), only when persona fields exist.
153
+
154
+ Returns an empty dict for non-persona runs so the metric-key set is unchanged
155
+ (additive, category-agnostic). When present:
156
+
157
+ - ``action_agreement`` = % of turns whose action is a reference action.
158
+ - ``reward_regret`` = mean per-turn ``reference_reward − model_reward``.
159
+ - ``pressure_weighted_agreement`` = pressure-weighted agreement %, so high-
160
+ danger turns count more (0.0 when total pressure is 0).
161
+ - ``persona_drift_turn`` = first turn where the rolling-mean (window 3)
162
+ agreement falls below 0.5, else 0.0 (never drifted).
163
+ """
164
+ scored = [t for t in turns if t.reference_actions is not None]
165
+ if not scored:
166
+ return {}
167
+
168
+ agreements = [1.0 if t.action in (t.reference_actions or []) else 0.0 for t in scored]
169
+ n = len(scored)
170
+ action_agreement = sum(agreements) / n * 100.0
171
+
172
+ regrets = [
173
+ (t.reference_reward - t.model_reward)
174
+ for t in scored
175
+ if t.reference_reward is not None and t.model_reward is not None
176
+ ]
177
+ reward_regret = sum(regrets) / len(regrets) if regrets else 0.0
178
+
179
+ pressures = [t.pressure if t.pressure is not None else 0.0 for t in scored]
180
+ total_pressure = sum(pressures)
181
+ if total_pressure > 0:
182
+ pressure_weighted = (
183
+ sum(a * p for a, p in zip(agreements, pressures)) / total_pressure * 100.0
184
+ )
185
+ else:
186
+ pressure_weighted = 0.0
187
+
188
+ drift_turn = 0.0
189
+ for i in range(n):
190
+ window = agreements[max(0, i - 2): i + 1]
191
+ if sum(window) / len(window) < 0.5:
192
+ drift_turn = float(scored[i].turn_idx)
193
+ break
194
+
195
+ return {
196
+ "action_agreement": action_agreement,
197
+ "reward_regret": reward_regret,
198
+ "pressure_weighted_agreement": pressure_weighted,
199
+ "persona_drift_turn": drift_turn,
200
+ }
proteus/runtime/session.py CHANGED
@@ -25,6 +25,12 @@ from proteus.grid.game import MotiveGridGame
25
  from proteus.grid.scenario import Scenario, get_scenario
26
  from proteus.runtime.memory import MemoryCheckpoint, render_memory_block
27
  from proteus.runtime.metrics import compute_metrics
 
 
 
 
 
 
28
  from proteus.runtime.rollout import optimal_rollout
29
  from proteus.runtime.trace import SessionTrace, TurnTrace
30
 
@@ -66,6 +72,7 @@ class SessionRunner:
66
  motive_category: str = "survival",
67
  memory: MemoryCheckpoint | None = None,
68
  memory_ref: str | None = None,
 
69
  ) -> None:
70
  self._scenario_name = scenario_name
71
  self._agent = agent
@@ -76,6 +83,7 @@ class SessionRunner:
76
  self._motive_category = motive_category
77
  self._memory = memory
78
  self._memory_ref = memory_ref
 
79
 
80
  self._scenario: Scenario | None = None
81
  self._game: MotiveGridGame | None = None
@@ -113,6 +121,16 @@ class SessionRunner:
113
  focal_pos = (focal.x, focal.y) if focal else (-1, -1)
114
  predator_pos = (predator.x, predator.y) if predator else (-1, -1)
115
  pre_bfs = self._scenario.safety_distance(self._game)
 
 
 
 
 
 
 
 
 
 
116
 
117
  result = self._agent.act(observation, list(_ACTIONS), system_prompt)
118
 
@@ -132,6 +150,15 @@ class SessionRunner:
132
  agent_distance_delta = self._scenario.agent_distance_delta(
133
  self._game, focal_pos, predator_pos
134
  )
 
 
 
 
 
 
 
 
 
135
 
136
  turns.append(
137
  TurnTrace(
@@ -162,6 +189,11 @@ class SessionRunner:
162
  pre_bfs_distance=pre_bfs,
163
  post_bfs_distance=post_bfs,
164
  agent_distance_delta=agent_distance_delta,
 
 
 
 
 
165
  )
166
  )
167
 
@@ -201,6 +233,9 @@ class SessionRunner:
201
  metrics=metrics,
202
  memory_ref=self._memory_ref,
203
  horizon=self._play_turns,
 
 
 
204
  )
205
 
206
  # ------------------------------------------------------------------ #
 
25
  from proteus.grid.scenario import Scenario, get_scenario
26
  from proteus.runtime.memory import MemoryCheckpoint, render_memory_block
27
  from proteus.runtime.metrics import compute_metrics
28
+ from proteus.runtime.persona import (
29
+ PersonaWeights,
30
+ pressure as persona_pressure,
31
+ reference_actions,
32
+ reward_rw,
33
+ )
34
  from proteus.runtime.rollout import optimal_rollout
35
  from proteus.runtime.trace import SessionTrace, TurnTrace
36
 
 
72
  motive_category: str = "survival",
73
  memory: MemoryCheckpoint | None = None,
74
  memory_ref: str | None = None,
75
+ persona: PersonaWeights | None = None,
76
  ) -> None:
77
  self._scenario_name = scenario_name
78
  self._agent = agent
 
83
  self._motive_category = motive_category
84
  self._memory = memory
85
  self._memory_ref = memory_ref
86
+ self._persona = persona
87
 
88
  self._scenario: Scenario | None = None
89
  self._game: MotiveGridGame | None = None
 
121
  focal_pos = (focal.x, focal.y) if focal else (-1, -1)
122
  predator_pos = (predator.x, predator.y) if predator else (-1, -1)
123
  pre_bfs = self._scenario.safety_distance(self._game)
124
+ # CP8 persona: reference action set + reference reward + pressure are
125
+ # read from the PRE-move state (the model never sees the weights).
126
+ ref_acts = ref_reward = turn_pressure = None
127
+ if self._persona is not None:
128
+ ref_acts = reference_actions(self._persona, self._scenario, self._game)
129
+ ref_reward = reward_rw(
130
+ self._persona, self._scenario, self._game,
131
+ focal_pos, predator_pos, ref_acts[0],
132
+ )
133
+ turn_pressure = persona_pressure(self._scenario, self._game)
134
 
135
  result = self._agent.act(observation, list(_ACTIONS), system_prompt)
136
 
 
150
  agent_distance_delta = self._scenario.agent_distance_delta(
151
  self._game, focal_pos, predator_pos
152
  )
153
+ # CP8 persona: the model's own R_w (BFS geometry is static, so the
154
+ # pre-move positions + actual blocked status fully determine it).
155
+ model_reward = reward_regret = None
156
+ if self._persona is not None:
157
+ model_reward = reward_rw(
158
+ self._persona, self._scenario, self._game,
159
+ focal_pos, predator_pos, result.action, blocked=blocked,
160
+ )
161
+ reward_regret = ref_reward - model_reward
162
 
163
  turns.append(
164
  TurnTrace(
 
189
  pre_bfs_distance=pre_bfs,
190
  post_bfs_distance=post_bfs,
191
  agent_distance_delta=agent_distance_delta,
192
+ reference_actions=ref_acts,
193
+ reference_reward=ref_reward,
194
+ model_reward=model_reward,
195
+ reward_regret=reward_regret,
196
+ pressure=turn_pressure,
197
  )
198
  )
199
 
 
233
  metrics=metrics,
234
  memory_ref=self._memory_ref,
235
  horizon=self._play_turns,
236
+ persona_weight_id=(
237
+ self._persona.persona_weight_id if self._persona else None
238
+ ),
239
  )
240
 
241
  # ------------------------------------------------------------------ #
proteus/runtime/trace.py CHANGED
@@ -79,6 +79,14 @@ class TurnTrace(BaseModel):
79
  pre_bfs_distance: int | None = None
80
  post_bfs_distance: int | None = None
81
  agent_distance_delta: float | None = None
 
 
 
 
 
 
 
 
82
 
83
 
84
  class SessionTrace(BaseModel):
@@ -121,3 +129,6 @@ class SessionTrace(BaseModel):
121
  turn_order: str = "focal_then_predator"
122
  capture_rule: str = "same_cell"
123
  horizon: int | None = None # = play_turns
 
 
 
 
79
  pre_bfs_distance: int | None = None
80
  post_bfs_distance: int | None = None
81
  agent_distance_delta: float | None = None
82
+ # CP8 persona-maintenance fields (spec §6.3/§7); set only when an eval runs
83
+ # against a hidden persona, else None. The weights are never serialized —
84
+ # only the public reference action set + scalar rewards/pressure.
85
+ reference_actions: list[str] | None = None
86
+ reference_reward: float | None = None
87
+ model_reward: float | None = None
88
+ reward_regret: float | None = None
89
+ pressure: float | None = None
90
 
91
 
92
  class SessionTrace(BaseModel):
 
129
  turn_order: str = "focal_then_predator"
130
  capture_rule: str = "same_cell"
131
  horizon: int | None = None # = play_turns
132
+ # CP8: public persona id when scored against a hidden persona (spec §7);
133
+ # the raw weights are never stored on the trace.
134
+ persona_weight_id: str | None = None
tests/runtime/test_persona.py CHANGED
@@ -30,3 +30,51 @@ def test_pressure_in_unit_range():
30
  s, g = _sg()
31
  p = pressure(s, g)
32
  assert 0.0 <= p <= 1.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
  s, g = _sg()
31
  p = pressure(s, g)
32
  assert 0.0 <= p <= 1.0
33
+
34
+
35
+ def _runner(action, persona_id="risk_averse", play_turns=1):
36
+ from proteus.providers import FakeProvider
37
+ from proteus.agents import VanillaAgent
38
+ from proteus.runtime import SessionRunner
39
+ from proteus.runtime.persona import get_persona
40
+ prov = FakeProvider([f"ACTION: {action}"] * 10, model_name="demo")
41
+ return SessionRunner(
42
+ "predator_evade", VanillaAgent(prov), difficulty=Difficulty.EASY,
43
+ seed=42, play_turns=play_turns, use_probe=False,
44
+ persona=get_persona(persona_id),
45
+ ).run()
46
+
47
+
48
+ def test_persona_run_records_id_and_fields():
49
+ trace = _runner("up")
50
+ assert trace.persona_weight_id == "risk_averse"
51
+ t = trace.turns[0]
52
+ assert t.reference_actions is not None
53
+ assert t.pressure is not None
54
+ assert t.model_reward is not None
55
+
56
+
57
+ def test_reference_action_yields_full_agreement_zero_regret():
58
+ # 'up' is a reference action at the EASY handover -> perfect maintenance.
59
+ m = _runner("up").metrics
60
+ assert m["action_agreement"] == 100.0
61
+ assert abs(m["reward_regret"]) < 1e-9
62
+
63
+
64
+ def test_into_wall_lowers_agreement_and_positive_regret():
65
+ # 'left' walks into the dead-end wall: not a reference action, worse reward.
66
+ m = _runner("left").metrics
67
+ assert m["action_agreement"] == 0.0
68
+ assert m["reward_regret"] > 0.0
69
+
70
+
71
+ def test_persona_metrics_absent_without_persona():
72
+ from proteus.providers import FakeProvider
73
+ from proteus.agents import VanillaAgent
74
+ from proteus.runtime import SessionRunner
75
+ prov = FakeProvider(["ACTION: up"] * 10, model_name="demo")
76
+ m = SessionRunner(
77
+ "predator_evade", VanillaAgent(prov), difficulty=Difficulty.EASY,
78
+ seed=42, play_turns=2, use_probe=False,
79
+ ).run().metrics
80
+ assert "action_agreement" not in m