| from agentic_workflow.config import LLM_MODEL |
| from langchain_ollama import ChatOllama |
| import re |
| import json |
| from langchain_core.messages import HumanMessage, SystemMessage |
|
|
| llm = ChatOllama(model=LLM_MODEL, temperature=0) |
|
|
| def evaluate_response(query: str, response: str) -> dict: |
| """ |
| Scores the draft response and decides whether to retry. |
| |
| Returns a dict with keys: |
| - score: int 1-10 |
| - pass: bool → True means use this response |
| - reason: str → short explanation |
| - suggestion: str → how to improve (if not passing) |
| """ |
| system = """ |
| You are a strict response quality evaluator. Given a user query and a draft response, return ONLY valid JSON: |
| - "score": integer 1-10 (10 = excellent) |
| - "pass": true if score >= 7, false otherwise |
| - "reason": one-sentence explanation |
| - "suggestion": if not passing, a short instruction to improve the response; else "" |
| |
| Return ONLY the JSON object. No explanation. |
| """ |
| |
| response_eval = llm.invoke([ |
| SystemMessage(content=system), |
| HumanMessage(content=f"Query: {query}\n\nDraft response: {response}") |
| ]) |
| raw = response_eval.content.strip() |
|
|
| raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw).strip() |
|
|
| try: |
| parsed = json.loads(raw) |
| except json.JSONDecodeError: |
| parsed = {} |
|
|
| return { |
| "score": 5, |
| "pass": False, |
| "reason": "Evaluation failed", |
| "suggestion": "Retry with more detail", |
| **parsed |
| } |
|
|
| |
| |
| |
| |
| |
|
|
| |
| |
| |
| |
| |
