src/experience/engine.py

"""Experience Engine for controlled, single-consumption learning loops."""

from __future__ import annotations

import math
from collections.abc import Callable, Mapping
from dataclasses import dataclass, field
from typing import Any

from src.embodiment.controlled import ControlledEmbodimentAgent
from src.embodiment.loop_contract import CycleContract, valid_tick
from src.embodiment.models import ActionCommand, EnvironmentObservation, SensorFrame
from src.embodiment.sensor import SensorAdapter
from src.embodiment.task_outcome import TaskOutcome, TaskOutcomeVerifier
from src.learning.learning_engine import LearningEngine
from src.memory import MemoryWorldModel
from src.memory.neural_episodic import NeuralEpisodicMemory
from src.profiles import BehaviorProfile

Encoder = Callable[[SensorFrame], Mapping[int, float]]
Decoder = Callable[[Any, SensorFrame], ActionCommand | tuple[ActionCommand, ...] | None]


@dataclass(frozen=True, slots=True)
class ExperienceStep:
    """Immutable audit record for one perception-action-feedback cycle."""

    tick: int
    frame: SensorFrame
    action: ActionCommand | None
    observation: EnvironmentObservation | None
    reward: float
    outcome: TaskOutcome | None = None


@dataclass(slots=True)
class ExperienceEngine:
    """Connect a controlled sensor loop to the real learning engine.

    Rewards are accepted only from environment observations. No language
    model, configuration value, or decoder output can write a reward.
    A consumed cycle cannot be retried after a downstream observer fails.
    """

    sensor: SensorAdapter
    network: Any
    encoder: Encoder
    decoder: Decoder
    embodiment: ControlledEmbodimentAgent
    learning: LearningEngine | None = None
    outcome_verifier: TaskOutcomeVerifier = field(default_factory=TaskOutcomeVerifier)
    memory: MemoryWorldModel | None = None
    neural_memory: NeuralEpisodicMemory | None = None
    behavior_profile: BehaviorProfile | None = None
    last_step: ExperienceStep | None = None
    _pending_frame: SensorFrame | None = None
    _pending_prediction: Any = None
    _cycle: CycleContract = field(default_factory=CycleContract, init=False)

    def reset(self, seed: int | None = None) -> EnvironmentObservation:
        """Reset the environment and episode-local cycle state, not safety stops."""

        self._abort_cycle()
        observation = self.embodiment.reset(seed)
        self.last_step = None
        self._cycle.reset()
        episode_id = f"episode-{self.embodiment.episode}"
        if self.memory is not None:
            self.memory.reset_episode(episode_id)
        if self.neural_memory is not None:
            self.neural_memory.reset_episode(episode_id)
        return observation

    def _abort_cycle(self) -> None:
        self._cycle.abort()
        self._pending_frame = None
        self._pending_prediction = None

    def step(self, tick: int) -> ExperienceStep:
        """Run one complete sensor, network, action, feedback and reward step."""

        self.prepare(tick)
        try:
            result = self.network.step()
        except Exception:
            self._abort_cycle()
            raise
        return self.complete(tick, result)

    def prepare(self, tick: int) -> SensorFrame:
        """Validate and encode input before exactly one existing runtime tick."""

        self._cycle.begin(tick)
        try:
            if not self.sensor.active:
                raise RuntimeError("experience sensor is inactive")
            frame = self.sensor.sample(tick)
            if (
                not valid_tick(frame.tick)
                or frame.tick != tick
                or frame.sensor_id != self.sensor.sensor_id
                or frame.modality != self.sensor.modality
            ):
                raise ValueError("sensor frame identity or tick mismatch")
            currents = dict(self.encoder(frame))
            if any(
                type(neuron_id) is not int
                or neuron_id < 0
                or isinstance(value, bool)
                or not math.isfinite(value)
                for neuron_id, value in currents.items()
            ):
                raise ValueError(
                    "encoder currents must have valid IDs and finite values"
                )
            self.network.inject_current_batch(currents)
            self._pending_frame = frame
            return frame
        except Exception:
            self._abort_cycle()
            raise

    def complete(self, tick: int, result: Any) -> ExperienceStep:
        """Consume before side effects; late failures cannot replay an action."""

        self._cycle.consume(tick)
        frame = self._pending_frame
        self._pending_frame = None
        self._pending_prediction = None
        if frame is None or frame.tick != tick:
            raise RuntimeError("complete() requires a matching prepare() call")
        observation = None
        decoded = self.decoder(result, frame)
        if isinstance(decoded, tuple):
            action: ActionCommand | None = (
                self.behavior_profile.select_action(decoded, tick=tick)
                if self.behavior_profile is not None
                else (decoded[0] if decoded else None)
            )
        else:
            action = decoded
        if action is not None and (not valid_tick(action.tick) or action.tick != tick):
            raise ValueError("action tick must match the prepared cycle")
        prediction = (
            None if self.memory is None else self.memory.predict(frame, action, tick)
        )
        if action is not None:
            observation = self.embodiment.step(action)
        outcome = (
            TaskOutcome(False, False, 0.0, "no environment observation")
            if observation is None
            else self.outcome_verifier.verify(observation)
        )
        reward = outcome.reward
        if not math.isfinite(reward):
            raise ValueError("environment reward must be finite")
        if self.learning is not None and observation is not None:
            self.learning.set_reward(reward, tick)
        record = ExperienceStep(tick, frame, action, observation, reward, outcome)
        if self.memory is not None:
            self.memory.complete(frame, action, observation, tick, prediction)
        if self.neural_memory is not None:
            self.neural_memory.record(result, frame, observation)
        if self.behavior_profile is not None:
            self.behavior_profile.update(success=outcome.success, tick=tick)
        self.last_step = record
        return record

    def attach_runtime(self, runtime: Any) -> None:
        """Attach to a RuntimeController without taking ownership of ticks."""

        runtime.add_pre_hook(self.prepare)
        runtime.add_hook(self.complete)


__all__ = ["ExperienceEngine", "ExperienceStep"]