Spaces:
Runtime error
Runtime error
Commit ·
d36047a
1
Parent(s): 45e0c57
feat(discovery): parse SELF: report + score self_correct in make_turn_trace
Browse files
proteus/game/runtime/_session_core.py
CHANGED
|
@@ -30,6 +30,7 @@ from proteus.game.metrics.persona import (
|
|
| 30 |
)
|
| 31 |
from proteus.game.metrics.rollout import optimal_rollout
|
| 32 |
from proteus.game.runtime.trace import SessionTrace, TurnTrace
|
|
|
|
| 33 |
|
| 34 |
_ACTIONS = ["up", "down", "left", "right", "stay"]
|
| 35 |
_DIRECTIONS = {"up", "down", "left", "right"}
|
|
@@ -245,6 +246,17 @@ def make_turn_trace(
|
|
| 245 |
blocked=blocked,
|
| 246 |
)
|
| 247 |
reward_regret = ref_reward - model_reward
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 248 |
return TurnTrace(
|
| 249 |
turn_idx=turn_idx,
|
| 250 |
observation=observation,
|
|
@@ -278,6 +290,8 @@ def make_turn_trace(
|
|
| 278 |
model_reward=model_reward,
|
| 279 |
reward_regret=reward_regret,
|
| 280 |
pressure=turn_pressure,
|
|
|
|
|
|
|
| 281 |
)
|
| 282 |
|
| 283 |
|
|
|
|
| 30 |
)
|
| 31 |
from proteus.game.metrics.rollout import optimal_rollout
|
| 32 |
from proteus.game.runtime.trace import SessionTrace, TurnTrace
|
| 33 |
+
from proteus.game.agents.parsing import extract_self_belief
|
| 34 |
|
| 35 |
_ACTIONS = ["up", "down", "left", "right", "stay"]
|
| 36 |
_DIRECTIONS = {"up", "down", "left", "right"}
|
|
|
|
| 246 |
blocked=blocked,
|
| 247 |
)
|
| 248 |
reward_regret = ref_reward - model_reward
|
| 249 |
+
# Find-your-body discovery: parse the optional SELF: report from raw_text
|
| 250 |
+
# and score it against the scenario's (hidden) true body index. No-op for
|
| 251 |
+
# non-discovery scenarios (discovery_candidates() == 0).
|
| 252 |
+
n_candidates = scenario.discovery_candidates()
|
| 253 |
+
self_belief = extract_self_belief(raw_text, n_candidates) if n_candidates else None
|
| 254 |
+
true_index = scenario.discovery_true_index()
|
| 255 |
+
self_correct = (
|
| 256 |
+
(self_belief == true_index)
|
| 257 |
+
if (self_belief is not None and true_index is not None)
|
| 258 |
+
else None
|
| 259 |
+
)
|
| 260 |
return TurnTrace(
|
| 261 |
turn_idx=turn_idx,
|
| 262 |
observation=observation,
|
|
|
|
| 290 |
model_reward=model_reward,
|
| 291 |
reward_regret=reward_regret,
|
| 292 |
pressure=turn_pressure,
|
| 293 |
+
self_belief=self_belief,
|
| 294 |
+
self_correct=self_correct,
|
| 295 |
)
|
| 296 |
|
| 297 |
|
tests/runtime/test_errand_discovery.py
CHANGED
|
@@ -33,3 +33,34 @@ def test_template_has_no_discovery():
|
|
| 33 |
scen = get_scenario("template")()
|
| 34 |
assert scen.discovery_candidates() == 0
|
| 35 |
assert scen.discovery_true_index() is None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
scen = get_scenario("template")()
|
| 34 |
assert scen.discovery_candidates() == 0
|
| 35 |
assert scen.discovery_true_index() is None
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
from proteus.game.runtime import _session_core as core
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def test_make_turn_trace_parses_correct_self_report():
|
| 42 |
+
scen, game = _errand()
|
| 43 |
+
true = scen.true_body_index
|
| 44 |
+
tt = core.make_turn_trace(
|
| 45 |
+
scen, game, turn_idx=1, observation="obs",
|
| 46 |
+
action="stay", raw_text=f"reasoning...\nSELF: {true}\nACTION: stay",
|
| 47 |
+
)
|
| 48 |
+
assert tt.self_belief == true
|
| 49 |
+
assert tt.self_correct is True
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def test_make_turn_trace_marks_wrong_self_report():
|
| 53 |
+
scen, game = _errand()
|
| 54 |
+
wrong = (scen.true_body_index + 1) % 3
|
| 55 |
+
tt = core.make_turn_trace(
|
| 56 |
+
scen, game, turn_idx=1, observation="obs",
|
| 57 |
+
action="stay", raw_text=f"SELF: {wrong}\nACTION: stay",
|
| 58 |
+
)
|
| 59 |
+
assert tt.self_belief == wrong and tt.self_correct is False
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def test_make_turn_trace_no_self_report_is_none():
|
| 63 |
+
scen, game = _errand()
|
| 64 |
+
tt = core.make_turn_trace(scen, game, turn_idx=1, observation="o",
|
| 65 |
+
action="stay", raw_text="ACTION: stay")
|
| 66 |
+
assert tt.self_belief is None and tt.self_correct is None
|