Spaces:
Paused
Paused
File size: 5,562 Bytes
5e0b58b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 | """Reward signals for Brain 5D three-factor plasticity.
This module provides the RewardSignal dataclass, which represents an
immutable scalar reward emitted at a specific simulation tick.
Reward signals are used for three-factor plasticity (reward-modulated STDP),
where a reward signal modulates weight changes based on eligibility traces.
The reward signal is:
- Immutable – Once created, it cannot be modified.
- Tick-aligned – Associated with a specific simulation tick.
- Finite – Validates that the reward value is finite.
Design Principles:
1. Rewards are data, not commands – They represent a scalar value at a tick.
2. Immutable – Safe for concurrent use and debugging.
3. Delayed – Can be checked for due-ness with a configurable delay.
Example:
>>> from src.learning.reward import RewardSignal, create_reward
>>> reward = create_reward(value=1.0, tick=100)
>>> reward.is_due(current_tick=105, delay_ticks=5)
True
>>> reward.is_due(current_tick=104, delay_ticks=5)
False
"""
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import Any
# ============================================================================
# Reward Signal
# ============================================================================
@dataclass(frozen=True, slots=True)
class RewardSignal:
"""Immutable scalar reward emitted at one simulation tick.
This dataclass represents a reward signal that can be used in
three-factor plasticity rules. It is associated with a specific
simulation tick and can be checked for due-ness with a delay.
Attributes:
value: The scalar reward value (must be finite).
tick: The simulation tick at which the reward was emitted.
Raises:
ValueError: If the reward value is not finite or the tick is negative.
Example:
>>> reward = RewardSignal(value=0.5, tick=100)
>>> reward.due_tick(delay_ticks=5)
105
"""
value: float
tick: int
def __post_init__(self) -> None:
"""Validate the reward signal after initialization."""
if not math.isfinite(self.value):
raise ValueError("reward value must be finite")
if self.tick < 0:
raise ValueError("reward tick must be >= 0")
def due_tick(self, delay_ticks: int) -> int:
"""Return the tick at which this reward becomes effective.
Args:
delay_ticks: The number of ticks to delay the reward (must be >= 0).
Returns:
The tick at which the reward is due.
Raises:
ValueError: If delay_ticks is negative.
Example:
>>> reward = RewardSignal(1.0, 100)
>>> reward.due_tick(5)
105
"""
if delay_ticks < 0:
raise ValueError(f"delay_ticks must be >= 0, got {delay_ticks}")
return self.tick + delay_ticks
def is_due(self, current_tick: int, delay_ticks: int) -> bool:
"""Return whether the configured reward delay has elapsed.
Args:
current_tick: The current simulation tick (must be >= 0).
delay_ticks: The number of ticks to delay the reward (must be >= 0).
Returns:
True if the reward is due at the current tick, False otherwise.
Raises:
ValueError: If current_tick is negative or delay_ticks is negative.
Example:
>>> reward = RewardSignal(1.0, 100)
>>> reward.is_due(current_tick=105, delay_ticks=5)
True
>>> reward.is_due(current_tick=104, delay_ticks=5)
False
"""
if current_tick < 0:
raise ValueError("current_tick must be >= 0")
if delay_ticks < 0:
raise ValueError("delay_ticks must be >= 0")
return current_tick >= self.due_tick(delay_ticks)
def to_dict(self) -> dict[str, Any]:
"""Convert to dictionary for serialization.
Returns:
A dictionary with the reward value and tick.
Example:
>>> reward = RewardSignal(1.0, 100)
>>> reward.to_dict()
{'value': 1.0, 'tick': 100}
"""
return {
"value": self.value,
"tick": self.tick,
}
def __repr__(self) -> str:
"""Return a string representation of the reward signal."""
return f"RewardSignal(value={self.value}, tick={self.tick})"
# ============================================================================
# Factory Function
# ============================================================================
def create_reward(value: float, tick: int) -> RewardSignal:
"""Create a reward signal with the given value and tick.
This is a convenience factory function for creating RewardSignal
instances.
Args:
value: The scalar reward value (must be finite).
tick: The simulation tick at which the reward is emitted (must be >= 0).
Returns:
A new RewardSignal instance.
Raises:
ValueError: If the value is not finite or the tick is negative.
Example:
>>> reward = create_reward(0.5, 100)
>>> print(reward)
RewardSignal(value=0.5, tick=100)
"""
return RewardSignal(value=value, tick=tick)
# ============================================================================
# Module Exports
# ============================================================================
__all__ = [
"RewardSignal",
"create_reward",
]
|