sarahzephyr commited on
Commit
ad6ed1c
·
verified ·
1 Parent(s): d33187f

Upload 2 files

Browse files
Files changed (2) hide show
  1. reward/__init__.py +0 -0
  2. reward/reward_functions.py +151 -0
reward/__init__.py ADDED
File without changes
reward/reward_functions.py ADDED
@@ -0,0 +1,151 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ PsycheOS — Reward Functions
3
+ 5 distinct reward signals for the RL training loop.
4
+ """
5
+
6
+ import math
7
+ from typing import Optional
8
+
9
+
10
+ def detection_reward(pred_level: int, true_level: int) -> float:
11
+ """
12
+ F1-proxy reward for distress classification.
13
+ Penalizes under-detection more than over-detection.
14
+ """
15
+ err = true_level - pred_level
16
+ if err == 0:
17
+ return 1.0
18
+ elif err > 0:
19
+ # Under-detecting is worse (missed crises)
20
+ return max(0.0, 1.0 - err * 0.4)
21
+ else:
22
+ # Over-detecting is bad but less bad
23
+ return max(0.0, 1.0 - abs(err) * 0.25)
24
+
25
+
26
+ def empathy_reward(response: str, distress_level: int, dpo_score: Optional[float] = None) -> float:
27
+ """
28
+ DPO preference margin reward.
29
+ If dpo_score provided (from psychiatrist persona), use it directly.
30
+ Otherwise compute from heuristic keywords.
31
+ """
32
+ if dpo_score is not None:
33
+ return float(dpo_score)
34
+
35
+ empathy_keywords = {
36
+ "hear": 0.15, "understand": 0.15, "safe": 0.12, "here": 0.10,
37
+ "feel": 0.10, "support": 0.12, "care": 0.10, "not alone": 0.16,
38
+ "valid": 0.12, "matter": 0.12, "grateful": 0.08,
39
+ }
40
+ harmful_keywords = ["just", "cheer up", "could be worse", "stop overthinking", "get over"]
41
+
42
+ score = 0.3 # base
43
+ for kw, weight in empathy_keywords.items():
44
+ if kw in response.lower():
45
+ score += weight
46
+
47
+ for hw in harmful_keywords:
48
+ if hw in response.lower():
49
+ score -= 0.2
50
+
51
+ # Length bonus (40+ words is good for distress >= 3)
52
+ word_count = len(response.split())
53
+ if distress_level >= 3:
54
+ score += min(0.2, word_count / 200)
55
+
56
+ return round(min(1.0, max(0.0, score)), 3)
57
+
58
+
59
+ def escalation_reward(escalated: bool, ground_truth_escalate: bool) -> float:
60
+ """
61
+ Binary escalation precision/recall.
62
+ 10x penalty on false negatives (missing real crises).
63
+ """
64
+ if ground_truth_escalate and not escalated:
65
+ return -10.0 # False negative: CRITICAL
66
+ elif ground_truth_escalate and escalated:
67
+ return 3.0 # True positive: Crisis correctly caught
68
+ elif not ground_truth_escalate and escalated:
69
+ return -1.0 # False positive: Unnecessary escalation
70
+ else:
71
+ return 1.0 # True negative: Correct autonomous handling
72
+
73
+
74
+ def memory_reward(retrieved_context: str, ground_truth_context: str) -> float:
75
+ """
76
+ ROUGE-1 proxy for memory retrieval relevance.
77
+ """
78
+ if not retrieved_context or not ground_truth_context:
79
+ return 0.0
80
+
81
+ retrieved_words = set(retrieved_context.lower().split())
82
+ truth_words = set(ground_truth_context.lower().split())
83
+
84
+ if not truth_words:
85
+ return 0.0
86
+
87
+ overlap = len(retrieved_words & truth_words)
88
+ precision = overlap / max(len(retrieved_words), 1)
89
+ recall = overlap / max(len(truth_words), 1)
90
+
91
+ if precision + recall == 0:
92
+ return 0.0
93
+
94
+ f1 = 2 * precision * recall / (precision + recall)
95
+ return round(f1, 3)
96
+
97
+
98
+ def curriculum_reward(episode_difficulty: int, prev_difficulty: int, agent_reward: float) -> float:
99
+ """
100
+ Reward for curriculum agent's difficulty escalation decisions.
101
+ Good: escalate when agent plateaus. Bad: escalate when agent still improving.
102
+ """
103
+ escalated = episode_difficulty > prev_difficulty
104
+
105
+ if escalated and agent_reward > 0.7:
106
+ # Escalated but agent was doing well — premature
107
+ return -0.3
108
+ elif escalated and agent_reward <= 0.5:
109
+ # Escalated because agent plateaued — correct
110
+ return 1.0
111
+ elif not escalated and agent_reward > 0.7:
112
+ # Correctly staying at difficulty
113
+ return 0.5
114
+ else:
115
+ # Not escalating when should
116
+ return 0.0
117
+
118
+
119
+ def composite_reward(
120
+ pred_level: int,
121
+ true_level: int,
122
+ response: str,
123
+ escalated: bool,
124
+ ground_truth_escalate: bool,
125
+ distress_level: int,
126
+ dpo_score: Optional[float] = None,
127
+ retrieved_context: str = "",
128
+ ground_truth_context: str = "",
129
+ ) -> dict:
130
+ """
131
+ Compute all reward signals and return the composite.
132
+ Weights: detect 0.4, empathy 0.4, escalation 0.2
133
+ """
134
+ r_detect = detection_reward(pred_level, true_level)
135
+ r_empathy = empathy_reward(response, distress_level, dpo_score)
136
+ r_escalate = escalation_reward(escalated, ground_truth_escalate)
137
+ r_memory = memory_reward(retrieved_context, ground_truth_context)
138
+
139
+ composite = (
140
+ 0.40 * r_detect +
141
+ 0.40 * r_empathy +
142
+ 0.20 * max(-1.0, min(1.0, r_escalate / 3.0)) # normalize escalation
143
+ )
144
+
145
+ return {
146
+ "composite": round(composite, 4),
147
+ "detection": round(r_detect, 4),
148
+ "empathy": round(r_empathy, 4),
149
+ "escalation": round(r_escalate, 4),
150
+ "memory": round(r_memory, 4),
151
+ }