Spaces:
Runtime error
Runtime error
Upload 2 files
Browse files- reward/__init__.py +0 -0
- reward/reward_functions.py +151 -0
reward/__init__.py
ADDED
|
File without changes
|
reward/reward_functions.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
PsycheOS — Reward Functions
|
| 3 |
+
5 distinct reward signals for the RL training loop.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
from typing import Optional
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def detection_reward(pred_level: int, true_level: int) -> float:
|
| 11 |
+
"""
|
| 12 |
+
F1-proxy reward for distress classification.
|
| 13 |
+
Penalizes under-detection more than over-detection.
|
| 14 |
+
"""
|
| 15 |
+
err = true_level - pred_level
|
| 16 |
+
if err == 0:
|
| 17 |
+
return 1.0
|
| 18 |
+
elif err > 0:
|
| 19 |
+
# Under-detecting is worse (missed crises)
|
| 20 |
+
return max(0.0, 1.0 - err * 0.4)
|
| 21 |
+
else:
|
| 22 |
+
# Over-detecting is bad but less bad
|
| 23 |
+
return max(0.0, 1.0 - abs(err) * 0.25)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def empathy_reward(response: str, distress_level: int, dpo_score: Optional[float] = None) -> float:
|
| 27 |
+
"""
|
| 28 |
+
DPO preference margin reward.
|
| 29 |
+
If dpo_score provided (from psychiatrist persona), use it directly.
|
| 30 |
+
Otherwise compute from heuristic keywords.
|
| 31 |
+
"""
|
| 32 |
+
if dpo_score is not None:
|
| 33 |
+
return float(dpo_score)
|
| 34 |
+
|
| 35 |
+
empathy_keywords = {
|
| 36 |
+
"hear": 0.15, "understand": 0.15, "safe": 0.12, "here": 0.10,
|
| 37 |
+
"feel": 0.10, "support": 0.12, "care": 0.10, "not alone": 0.16,
|
| 38 |
+
"valid": 0.12, "matter": 0.12, "grateful": 0.08,
|
| 39 |
+
}
|
| 40 |
+
harmful_keywords = ["just", "cheer up", "could be worse", "stop overthinking", "get over"]
|
| 41 |
+
|
| 42 |
+
score = 0.3 # base
|
| 43 |
+
for kw, weight in empathy_keywords.items():
|
| 44 |
+
if kw in response.lower():
|
| 45 |
+
score += weight
|
| 46 |
+
|
| 47 |
+
for hw in harmful_keywords:
|
| 48 |
+
if hw in response.lower():
|
| 49 |
+
score -= 0.2
|
| 50 |
+
|
| 51 |
+
# Length bonus (40+ words is good for distress >= 3)
|
| 52 |
+
word_count = len(response.split())
|
| 53 |
+
if distress_level >= 3:
|
| 54 |
+
score += min(0.2, word_count / 200)
|
| 55 |
+
|
| 56 |
+
return round(min(1.0, max(0.0, score)), 3)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def escalation_reward(escalated: bool, ground_truth_escalate: bool) -> float:
|
| 60 |
+
"""
|
| 61 |
+
Binary escalation precision/recall.
|
| 62 |
+
10x penalty on false negatives (missing real crises).
|
| 63 |
+
"""
|
| 64 |
+
if ground_truth_escalate and not escalated:
|
| 65 |
+
return -10.0 # False negative: CRITICAL
|
| 66 |
+
elif ground_truth_escalate and escalated:
|
| 67 |
+
return 3.0 # True positive: Crisis correctly caught
|
| 68 |
+
elif not ground_truth_escalate and escalated:
|
| 69 |
+
return -1.0 # False positive: Unnecessary escalation
|
| 70 |
+
else:
|
| 71 |
+
return 1.0 # True negative: Correct autonomous handling
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def memory_reward(retrieved_context: str, ground_truth_context: str) -> float:
|
| 75 |
+
"""
|
| 76 |
+
ROUGE-1 proxy for memory retrieval relevance.
|
| 77 |
+
"""
|
| 78 |
+
if not retrieved_context or not ground_truth_context:
|
| 79 |
+
return 0.0
|
| 80 |
+
|
| 81 |
+
retrieved_words = set(retrieved_context.lower().split())
|
| 82 |
+
truth_words = set(ground_truth_context.lower().split())
|
| 83 |
+
|
| 84 |
+
if not truth_words:
|
| 85 |
+
return 0.0
|
| 86 |
+
|
| 87 |
+
overlap = len(retrieved_words & truth_words)
|
| 88 |
+
precision = overlap / max(len(retrieved_words), 1)
|
| 89 |
+
recall = overlap / max(len(truth_words), 1)
|
| 90 |
+
|
| 91 |
+
if precision + recall == 0:
|
| 92 |
+
return 0.0
|
| 93 |
+
|
| 94 |
+
f1 = 2 * precision * recall / (precision + recall)
|
| 95 |
+
return round(f1, 3)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def curriculum_reward(episode_difficulty: int, prev_difficulty: int, agent_reward: float) -> float:
|
| 99 |
+
"""
|
| 100 |
+
Reward for curriculum agent's difficulty escalation decisions.
|
| 101 |
+
Good: escalate when agent plateaus. Bad: escalate when agent still improving.
|
| 102 |
+
"""
|
| 103 |
+
escalated = episode_difficulty > prev_difficulty
|
| 104 |
+
|
| 105 |
+
if escalated and agent_reward > 0.7:
|
| 106 |
+
# Escalated but agent was doing well — premature
|
| 107 |
+
return -0.3
|
| 108 |
+
elif escalated and agent_reward <= 0.5:
|
| 109 |
+
# Escalated because agent plateaued — correct
|
| 110 |
+
return 1.0
|
| 111 |
+
elif not escalated and agent_reward > 0.7:
|
| 112 |
+
# Correctly staying at difficulty
|
| 113 |
+
return 0.5
|
| 114 |
+
else:
|
| 115 |
+
# Not escalating when should
|
| 116 |
+
return 0.0
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def composite_reward(
|
| 120 |
+
pred_level: int,
|
| 121 |
+
true_level: int,
|
| 122 |
+
response: str,
|
| 123 |
+
escalated: bool,
|
| 124 |
+
ground_truth_escalate: bool,
|
| 125 |
+
distress_level: int,
|
| 126 |
+
dpo_score: Optional[float] = None,
|
| 127 |
+
retrieved_context: str = "",
|
| 128 |
+
ground_truth_context: str = "",
|
| 129 |
+
) -> dict:
|
| 130 |
+
"""
|
| 131 |
+
Compute all reward signals and return the composite.
|
| 132 |
+
Weights: detect 0.4, empathy 0.4, escalation 0.2
|
| 133 |
+
"""
|
| 134 |
+
r_detect = detection_reward(pred_level, true_level)
|
| 135 |
+
r_empathy = empathy_reward(response, distress_level, dpo_score)
|
| 136 |
+
r_escalate = escalation_reward(escalated, ground_truth_escalate)
|
| 137 |
+
r_memory = memory_reward(retrieved_context, ground_truth_context)
|
| 138 |
+
|
| 139 |
+
composite = (
|
| 140 |
+
0.40 * r_detect +
|
| 141 |
+
0.40 * r_empathy +
|
| 142 |
+
0.20 * max(-1.0, min(1.0, r_escalate / 3.0)) # normalize escalation
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
return {
|
| 146 |
+
"composite": round(composite, 4),
|
| 147 |
+
"detection": round(r_detect, 4),
|
| 148 |
+
"empathy": round(r_empathy, 4),
|
| 149 |
+
"escalation": round(r_escalate, 4),
|
| 150 |
+
"memory": round(r_memory, 4),
|
| 151 |
+
}
|