Download purpose_agent/purpose_function.py from Rohan03/purpose-agent: direct link, hf CLI and curl.
- Browser
- Download file 17.1 kB
-
https://huggingface.co/Rohan03/purpose-agent/resolve/fedfb2ecc74ce3ec74f265cfebdb15e5fcbeabdc/purpose_agent/purpose_function.py
- Command line
-
hf download hf://Rohan03/purpose-agent@fedfb2ecc74ce3ec74f265cfebdb15e5fcbeabdc/purpose_agent/purpose_function.py
-
curl -L -o purpose_function.py https://huggingface.co/Rohan03/purpose-agent/resolve/fedfb2ecc74ce3ec74f265cfebdb15e5fcbeabdc/purpose_agent/purpose_function.py
17.1 kB
| """ | |
| Purpose Function — The Critic / State Evaluator. | |
| This is the core innovation: a strictly separated LLM call that evaluates | |
| state improvement Φ(s). It rewards the agent ONLY if Φ(s_new) > Φ(s_current). | |
| Design principles (from literature): | |
| 1. Score AFTER environment feedback, never from expected state alone (LATS) | |
| 2. Require specific observable state changes as evidence (SPC anti-hacking) | |
| 3. Use separate LLM call / separate system prompt from the Actor (MUSE) | |
| 4. Normalize scores to prevent inflation over trajectory (novel addition) | |
| 5. V(s) = λ·LM_score + (1-λ)·consistency_score (LATS formulation) | |
| The Purpose Function is intentionally "non-hackable" by design: | |
| - It sees the ACTUAL new state, not the Actor's prediction | |
| - It must cite specific evidence for every score | |
| - Scores are bounded and normalized | |
| - The system prompt explicitly guards against sycophancy and vague reasoning | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import logging | |
| from typing import Any | |
| from purpose_agent.types import Action, PurposeScore, State | |
| from purpose_agent.llm_backend import ChatMessage, LLMBackend | |
| logger = logging.getLogger(__name__) | |
| # --------------------------------------------------------------------------- | |
| # Purpose Function System Prompt — The "Non-Hackable Judge" | |
| # --------------------------------------------------------------------------- | |
| PURPOSE_FUNCTION_SYSTEM_PROMPT = """\ | |
| You are a STATE EVALUATOR — a strict, impartial judge of progress toward a goal. | |
| You are NOT the agent. You do NOT help the agent. You ONLY measure progress. | |
| ## Your Role | |
| Given a state transition (state_before → action → state_after) and an ultimate purpose, | |
| you compute two scores: | |
| - Φ(state_before): How far the OLD state was from the purpose (0.0 = no progress, 10.0 = goal achieved) | |
| - Φ(state_after): How far the NEW state is from the purpose (same scale) | |
| The delta Φ(state_after) - Φ(state_before) is the ONLY signal the agent receives. | |
| ## STRICT RULES — Violation of any rule invalidates your evaluation | |
| 1. **EVIDENCE REQUIRED**: Every score MUST cite a specific, observable change in the | |
| state data. "The state improved" is NOT evidence. "Field 'score' changed from 3 to 7" | |
| IS evidence. If you cannot cite a specific change, the delta MUST be 0.0. | |
| 2. **NO CREDIT FOR INTENTIONS**: The agent's "thought" and "expected_delta" are | |
| provided for context only. You score based on ACTUAL state changes, never on | |
| what the agent intended or claimed would happen. | |
| 3. **NO SYCOPHANCY**: You are not the agent's friend. Do not inflate scores to be | |
| encouraging. A lateral move (no improvement) gets delta = 0.0. A regression gets | |
| negative delta. Be precise. | |
| 4. **MONOTONIC SCALE**: Φ = 0.0 means the state has zero progress toward the purpose. | |
| Φ = 10.0 means the purpose is fully achieved. Intermediate values are proportional. | |
| Justify WHY you chose each specific value. | |
| 5. **ANTI-GAMING**: If the action appears to manipulate the state in a way that | |
| superficially looks like progress but doesn't genuinely advance the purpose | |
| (e.g., changing a label without doing the work), score it as delta = 0.0 or negative | |
| and flag it in your evidence field. | |
| 6. **CONSISTENCY**: If a state identical to one you scored before appears again, | |
| it MUST receive the same Φ score. Progress is objective, not relative to your mood. | |
| 7. **CONFIDENCE**: Rate your confidence 0.0–1.0. High confidence (>0.8) requires | |
| clear, unambiguous evidence. If the state change is ambiguous, lower your confidence. | |
| ## Scoring Guide | |
| - Φ = 0.0: No meaningful progress toward the purpose | |
| - Φ = 1.0–3.0: Initial setup/preparation steps completed | |
| - Φ = 4.0–6.0: Substantive progress, key sub-goals partially achieved | |
| - Φ = 7.0–8.0: Most of the purpose is achieved, final steps remaining | |
| - Φ = 9.0: Purpose essentially achieved with minor polish needed | |
| - Φ = 10.0: Purpose fully and completely achieved | |
| """ | |
| PURPOSE_FUNCTION_EVAL_PROMPT = """\ | |
| ## Ultimate Purpose | |
| {purpose} | |
| ## State BEFORE Action | |
| {state_before} | |
| ## Action Taken | |
| Name: {action_name} | |
| Parameters: {action_params} | |
| Agent's Thought: {action_thought} | |
| Agent's Prediction: {expected_delta} | |
| ## State AFTER Action (this is the ACTUAL result — score based on THIS) | |
| {state_after} | |
| Evaluate this state transition. Remember: | |
| - Score Φ(state_before) and Φ(state_after) on the 0.0–10.0 scale | |
| - Cite SPECIFIC evidence from the state data | |
| - Do NOT give credit for intentions — only actual changes | |
| """ | |
| # --------------------------------------------------------------------------- | |
| # Purpose Function Schema (for structured output) | |
| # --------------------------------------------------------------------------- | |
| PURPOSE_SCORE_SCHEMA: dict[str, Any] = { | |
| "type": "object", | |
| "properties": { | |
| "phi_before": { | |
| "type": "number", | |
| "minimum": 0.0, | |
| "maximum": 10.0, | |
| "description": "Φ(state_before) — distance-to-purpose of the state before the action", | |
| }, | |
| "phi_after": { | |
| "type": "number", | |
| "minimum": 0.0, | |
| "maximum": 10.0, | |
| "description": "Φ(state_after) — distance-to-purpose of the state after the action", | |
| }, | |
| "reasoning": { | |
| "type": "string", | |
| "description": "Step-by-step justification for both scores (max 200 words)", | |
| }, | |
| "evidence": { | |
| "type": "string", | |
| "description": "Specific observable state changes that justify the delta (REQUIRED)", | |
| }, | |
| "confidence": { | |
| "type": "number", | |
| "minimum": 0.0, | |
| "maximum": 1.0, | |
| "description": "Confidence in this evaluation (0.0 = pure guess, 1.0 = certain)", | |
| }, | |
| }, | |
| "required": ["phi_before", "phi_after", "reasoning", "evidence", "confidence"], | |
| } | |
| # --------------------------------------------------------------------------- | |
| # Purpose Function Class | |
| # --------------------------------------------------------------------------- | |
| class PurposeFunction: | |
| """ | |
| The Critic — evaluates state transitions via Φ(s) scoring. | |
| Uses a SEPARATE LLM call from the Actor to prevent self-confirmation bias | |
| (per MUSE's Reflect Agent design, arxiv:2510.08002). | |
| Can optionally use a different model than the Actor (recommended for | |
| production — use a stronger model as the critic). | |
| Args: | |
| llm: LLM backend (can be same or different from Actor's) | |
| score_cache_size: Max entries in the Φ score cache (for consistency) | |
| require_evidence: If True, reject scores with empty evidence | |
| min_confidence: Minimum confidence threshold — below this, score is discarded | |
| """ | |
| def __init__( | |
| self, | |
| llm: LLMBackend, | |
| score_cache_size: int = 1000, | |
| require_evidence: bool = True, | |
| min_confidence: float = 0.3, | |
| ): | |
| self.llm = llm | |
| self.require_evidence = require_evidence | |
| self.min_confidence = min_confidence | |
| # Cache: state_hash → Φ score (for consistency rule #6) | |
| self._phi_cache: dict[str, float] = {} | |
| self._cache_size = score_cache_size | |
| # Running stats for normalization | |
| self._score_history: list[float] = [] | |
| # ------------------------------------------------------------------ | |
| # Core Evaluation | |
| # ------------------------------------------------------------------ | |
| def evaluate( | |
| self, | |
| state_before: State, | |
| action: Action, | |
| state_after: State, | |
| purpose: str, | |
| ) -> PurposeScore: | |
| """ | |
| Evaluate a state transition: did the action move closer to the purpose? | |
| Returns a PurposeScore with phi_before, phi_after, delta, reasoning, | |
| evidence, and confidence. | |
| Anti-hacking measures: | |
| 1. Scores based on ACTUAL state_after (not actor's expected_delta) | |
| 2. Evidence is required — vague scores are rejected | |
| 3. Cached Φ values enforce consistency | |
| 4. Confidence threshold filters uncertain evaluations | |
| """ | |
| # Check cache for consistency (Rule #6) | |
| cached_before = self._get_cached_phi(state_before) | |
| cached_after = self._get_cached_phi(state_after) | |
| # Build evaluation prompt | |
| messages = [ | |
| ChatMessage(role="system", content=PURPOSE_FUNCTION_SYSTEM_PROMPT), | |
| ChatMessage(role="user", content=PURPOSE_FUNCTION_EVAL_PROMPT.format( | |
| purpose=purpose, | |
| state_before=state_before.describe(), | |
| state_after=state_after.describe(), | |
| action_name=action.name, | |
| action_params=json.dumps(action.params, default=str), | |
| action_thought=action.thought, | |
| expected_delta=action.expected_delta, | |
| )), | |
| ] | |
| # Get structured evaluation from LLM | |
| try: | |
| raw_score = self.llm.generate_structured( | |
| messages, schema=PURPOSE_SCORE_SCHEMA, temperature=0.2 | |
| ) | |
| except Exception as e: | |
| logger.error(f"Purpose Function structured output failed: {e}") | |
| # Fall back to text-based evaluation | |
| raw_score = self._fallback_evaluate(messages) | |
| # Extract and validate scores | |
| phi_before = float(raw_score.get("phi_before", 0.0)) | |
| phi_after = float(raw_score.get("phi_after", 0.0)) | |
| reasoning = str(raw_score.get("reasoning", "")) | |
| evidence = str(raw_score.get("evidence", "")) | |
| confidence = float(raw_score.get("confidence", 0.5)) | |
| # Clamp to valid range | |
| phi_before = max(0.0, min(10.0, phi_before)) | |
| phi_after = max(0.0, min(10.0, phi_after)) | |
| confidence = max(0.0, min(1.0, confidence)) | |
| # Apply anti-hacking rules | |
| phi_before, phi_after, confidence = self._apply_safeguards( | |
| phi_before, phi_after, evidence, confidence, | |
| cached_before, cached_after, | |
| ) | |
| delta = phi_after - phi_before | |
| # Update caches | |
| self._cache_phi(state_before, phi_before) | |
| self._cache_phi(state_after, phi_after) | |
| self._score_history.append(phi_after) | |
| score = PurposeScore( | |
| phi_before=phi_before, | |
| phi_after=phi_after, | |
| delta=delta, | |
| reasoning=reasoning, | |
| evidence=evidence, | |
| confidence=confidence, | |
| ) | |
| logger.info( | |
| f"Purpose Function: Φ({phi_before:.1f}) → Φ({phi_after:.1f}), " | |
| f"Δ={delta:+.2f}, conf={confidence:.2f}, improved={score.improved}" | |
| ) | |
| return score | |
| # ------------------------------------------------------------------ | |
| # Anti-Hacking Safeguards | |
| # ------------------------------------------------------------------ | |
| def _apply_safeguards( | |
| self, | |
| phi_before: float, | |
| phi_after: float, | |
| evidence: str, | |
| confidence: float, | |
| cached_before: float | None, | |
| cached_after: float | None, | |
| ) -> tuple[float, float, float]: | |
| """ | |
| Apply anti-reward-hacking safeguards. | |
| 1. Evidence requirement: no evidence → delta forced to 0 | |
| 2. Cache consistency: if we've scored this state before, use cached value | |
| 3. Confidence threshold: low confidence → reduce delta magnitude | |
| 4. Anomaly detection: suspiciously large jumps get confidence penalty | |
| """ | |
| # Rule 1: Require evidence | |
| if self.require_evidence and len(evidence.strip()) < 10: | |
| logger.warning("Purpose Function: Insufficient evidence, forcing delta=0") | |
| phi_after = phi_before # No credit without evidence | |
| confidence = max(confidence, 0.1) | |
| # Rule 2: Cache consistency (allow small drift for scoring noise) | |
| if cached_before is not None: | |
| drift = abs(phi_before - cached_before) | |
| if drift > 1.0: | |
| logger.warning( | |
| f"Purpose Function: Inconsistent Φ_before " | |
| f"(new={phi_before:.1f}, cached={cached_before:.1f}), " | |
| f"using cached value" | |
| ) | |
| phi_before = cached_before | |
| if cached_after is not None: | |
| drift = abs(phi_after - cached_after) | |
| if drift > 1.0: | |
| logger.warning( | |
| f"Purpose Function: Inconsistent Φ_after " | |
| f"(new={phi_after:.1f}, cached={cached_after:.1f}), " | |
| f"using cached value" | |
| ) | |
| phi_after = cached_after | |
| # Rule 3: Confidence threshold | |
| if confidence < self.min_confidence: | |
| logger.warning( | |
| f"Purpose Function: Low confidence ({confidence:.2f}), " | |
| f"reducing delta magnitude by 50%" | |
| ) | |
| midpoint = (phi_before + phi_after) / 2 | |
| phi_after = midpoint + (phi_after - midpoint) * 0.5 | |
| # Rule 4: Anomaly detection — flag suspiciously large single-step jumps | |
| delta = phi_after - phi_before | |
| if abs(delta) > 3.0: | |
| logger.warning( | |
| f"Purpose Function: Unusually large delta ({delta:+.1f}), " | |
| f"applying confidence penalty" | |
| ) | |
| confidence = min(confidence, 0.5) | |
| return phi_before, phi_after, confidence | |
| # ------------------------------------------------------------------ | |
| # Caching | |
| # ------------------------------------------------------------------ | |
| def _state_hash(self, state: State) -> str: | |
| """Hash a state for cache lookup (based on data content).""" | |
| return json.dumps(state.data, sort_keys=True, default=str) | |
| def _get_cached_phi(self, state: State) -> float | None: | |
| return self._phi_cache.get(self._state_hash(state)) | |
| def _cache_phi(self, state: State, phi: float) -> None: | |
| key = self._state_hash(state) | |
| if len(self._phi_cache) >= self._cache_size: | |
| # Evict oldest (FIFO — good enough for our use case) | |
| oldest_key = next(iter(self._phi_cache)) | |
| del self._phi_cache[oldest_key] | |
| self._phi_cache[key] = phi | |
| # ------------------------------------------------------------------ | |
| # Normalization (prevent score inflation over long trajectories) | |
| # ------------------------------------------------------------------ | |
| def get_normalized_phi(self, raw_phi: float) -> float: | |
| """ | |
| Normalize a Φ score relative to the trajectory's score distribution. | |
| Prevents the common failure mode where LLM scores drift upward over | |
| a trajectory regardless of actual progress. | |
| """ | |
| if len(self._score_history) < 3: | |
| return raw_phi | |
| mean = sum(self._score_history) / len(self._score_history) | |
| variance = sum((x - mean) ** 2 for x in self._score_history) / len(self._score_history) | |
| std = max(variance ** 0.5, 0.1) # Avoid division by zero | |
| # Z-score normalization mapped back to 0-10 | |
| z = (raw_phi - mean) / std | |
| normalized = 5.0 + z * 2.0 # Center at 5, spread by 2 | |
| return max(0.0, min(10.0, normalized)) | |
| def reset_trajectory_stats(self) -> None: | |
| """Reset per-trajectory normalization stats. Call at trajectory start.""" | |
| self._score_history = [] | |
| # ------------------------------------------------------------------ | |
| # Fallback | |
| # ------------------------------------------------------------------ | |
| def _fallback_evaluate(self, messages: list[ChatMessage]) -> dict[str, Any]: | |
| """Text-based fallback when structured output is unavailable.""" | |
| raw = self.llm.generate(messages, temperature=0.2) | |
| import re | |
| phi_before = 0.0 | |
| phi_after = 0.0 | |
| # Try to extract scores from text | |
| before_match = re.search(r'[Φφ]\s*\(?state_?before\)?\s*[=:]\s*([\d.]+)', raw, re.IGNORECASE) | |
| after_match = re.search(r'[Φφ]\s*\(?state_?after\)?\s*[=:]\s*([\d.]+)', raw, re.IGNORECASE) | |
| if before_match: | |
| phi_before = float(before_match.group(1)) | |
| if after_match: | |
| phi_after = float(after_match.group(1)) | |
| # Also try "Score: X/10" patterns | |
| if not before_match: | |
| score_matches = re.findall(r'(\d+\.?\d*)\s*/?\s*10', raw) | |
| if len(score_matches) >= 2: | |
| phi_before = float(score_matches[0]) | |
| phi_after = float(score_matches[1]) | |
| elif len(score_matches) == 1: | |
| phi_after = float(score_matches[0]) | |
| confidence_match = re.search(r'confidence\s*[=:]\s*([\d.]+)', raw, re.IGNORECASE) | |
| confidence = float(confidence_match.group(1)) if confidence_match else 0.4 | |
| return { | |
| "phi_before": phi_before, | |
| "phi_after": phi_after, | |
| "reasoning": raw[:500], | |
| "evidence": raw[500:800] if len(raw) > 500 else "", | |
| "confidence": confidence, | |
| } | |