shashankheg commited on
Commit
a108ce9
Β·
1 Parent(s): b4c8380

Add comprehensive evaluation pipeline

Browse files
Files changed (1) hide show
  1. src/evaluation.py +481 -0
src/evaluation.py ADDED
@@ -0,0 +1,481 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import json
3
+ import time
4
+ from typing import Dict, List, Any
5
+ from dotenv import load_dotenv
6
+ from src.utils.llm import get_fast_llm
7
+
8
+ load_dotenv()
9
+
10
+ from src.features.translation import translate_text
11
+ from src.features.summarization import summarize_text
12
+ from src.features.keyword_extraction import extract_keywords
13
+ from src.features.email_writer import write_email, write_cover_letter
14
+ from src.graphs.chat_graph import ChatSession
15
+
16
+
17
+
18
+ # ── SAMPLE TEST DATA ────────────────────────────────────────────────────────
19
+
20
+
21
+ TRANSLATION_TESTS = [
22
+ {
23
+ "text": "Artificial intelligence is transforming the world.",
24
+ "source": "English", "target": "French",
25
+ "reference": "L'intelligence artificielle transforme le monde."
26
+ },
27
+
28
+ {
29
+ "text": "Machine learning enables computers to learn from data.",
30
+ "source": "English", "target": "Spanish",
31
+ "reference": "El aprendizaje automΓ‘tico permite a las computadoras aprender de los datos."
32
+ },
33
+
34
+ {
35
+ "text": "Deep learning uses neural networks to solve complex problems.",
36
+ "source": "English", "target": "German",
37
+ "reference": "Deep Learning verwendet neuronale Netze, um komplexe Probleme zu lΓΆsen."
38
+ },
39
+
40
+
41
+ ]
42
+
43
+
44
+ SUMMARIZATION_TESTS = [
45
+
46
+ {"text": """Artificial intelligence (AI) is intelligence demonstrated by machines,
47
+ as opposed to the natural intelligence displayed by animals including humans.
48
+ AI research has been defined as the field of study of intelligent agents,
49
+ which refers to any system that perceives its environment and takes actions
50
+ that maximize its chance of achieving its goals. The term "artificial intelligence"
51
+ had previously been used to describe machines that mimic and display human cognitive
52
+ skills associated with the human mind, such as learning and problem-solving.
53
+ This definition has since been rejected by major AI researchers who now describe
54
+ AI in terms of rationality and acting rationally, which does not limit how
55
+ intelligence can be articulated. AI applications include advanced web search engines,
56
+ recommendation systems, understanding human speech, self-driving cars, generative
57
+ tools and competing at the highest level in strategic games.""",
58
+ "reference": "AI is machine-demonstrated intelligence that perceives environments and takes goal-maximizing actions, with applications ranging from search engines to self-driving cars."
59
+ },
60
+ {
61
+ "text": """LangChain is a framework designed to simplify the creation of applications
62
+ using large language models. As a language model integration framework, LangChain's
63
+ use-cases largely overlap with those of language models in general, including document
64
+ analysis and summarization, chatbots, and code analysis. LangChain was launched in
65
+ October 2022 as an open source project. It provides a standard interface for chains,
66
+ lots of integrations with other tools, and end-to-end chains for common applications.
67
+ The framework allows AI developers to develop applications based on combining LLMs
68
+ with other sources of computation or knowledge.""",
69
+ "reference": "LangChain is an open-source framework that simplifies building LLM applications through standard interfaces, tool integrations, and pre-built chains."
70
+ },
71
+ ]
72
+
73
+
74
+ KEYWORD_TESTS = [
75
+ {
76
+ "text": """Natural language processing (NLP) is a subfield of linguistics,
77
+ computer science, and artificial intelligence concerned with the interactions
78
+ between computers and human language, in particular how to program computers
79
+ to process and analyze large amounts of natural language data. The goal is
80
+ a computer capable of understanding the contents of documents, including the
81
+ contextual nuances of the language within them.""",
82
+ "expected_keywords": ["natural language processing", "NLP", "artificial intelligence",
83
+ "computer science", "linguistics", "language", "computers"]
84
+ },
85
+ ]
86
+
87
+
88
+ EMAIL_TESTS = [
89
+ {
90
+ "type": "follow-up",
91
+ "context": "Following up on a job application for ML Engineer position submitted last week",
92
+ "tone": "professional",
93
+ "recipient": "Hiring Manager",
94
+ "sender": "Shashank Hegde",
95
+ "check_words": ["follow", "application", "ML Engineer", "Shashank"]
96
+ },
97
+
98
+ {
99
+ "type": "thank you",
100
+ "context": "Thanking the interviewer after a data science interview",
101
+ "tone": "professional",
102
+ "recipient": "Dr. Smith",
103
+ "sender": "Shashank",
104
+ "check_words": ["thank", "interview", "opportunity"]
105
+ },
106
+ ]
107
+
108
+ CHAT_TESTS = [
109
+ {"input": "What is machine learning?", "check": ["learning", "data", "model", "algorithm"]},
110
+ {"input": "Explain neural networks in simple terms.", "check": ["network", "neuron", "layer", "brain"]},
111
+ {"input": "What are the differences between supervised and unsupervised learning?",
112
+ "check": ["supervised", "unsupervised", "label", "data"]},
113
+ ]
114
+
115
+ COVER_LETTER_TESTS = [
116
+ {
117
+ "job": "ML Engineer",
118
+ "company": "Google DeepMind",
119
+ "skills": "Python, PyTorch, LangChain, MLflow",
120
+ "experience": "3 years building ML pipelines and deploying models to production",
121
+ "name": "Shashank Hegde",
122
+ "check_words": ["ML Engineer", "Google", "Python", "Shashank"]
123
+ }
124
+ ]
125
+
126
+
127
+ # ── UTILITY FUNCTIONS ────────────────────────────────────────────────────────
128
+
129
+ def simple_bleu( hypothesis:str, reference:str) -> float:
130
+ """Simple word-overlap BLEU approximation (no external library needed)."""
131
+ hyp_words = set(hypothesis.lower().split())
132
+ ref_words = set(reference.lower().split())
133
+
134
+ if not ref_words:
135
+ return 0.0
136
+
137
+ overlap = hyp_words & ref_words
138
+ return len(overlap) / len(ref_words)
139
+
140
+
141
+ def rouge_L(hypothesis:str, reference:str) -> float:
142
+ """Simple ROUGE-L approximation (no external library needed)."""
143
+ # This is a simplified version of ROUGE-L, which measures the longest common subsequence
144
+ # between the hypothesis and reference.
145
+ hyp = list(hypothesis.lower())
146
+ ref = list(reference.lower())
147
+
148
+ if not ref:
149
+ return 0.0
150
+
151
+ common = set(hyp) & set(ref)
152
+ precision = len(common) / len(hyp) if hyp else 0
153
+ recall = len(common) / len(ref) if ref else 0
154
+ if precision + recall == 0:
155
+ return 0.0
156
+ return 2 * (precision * recall) / (precision + recall)
157
+
158
+ def keyword_precision(extracted: str, expected: List[str], k: int = 10) -> float:
159
+ """Check how many expected keywords appear in extracted output."""
160
+ extracted_lower = extracted.lower()
161
+ found = sum(1 for kw in expected if kw.lower() in extracted_lower)
162
+ return found / len(expected)
163
+
164
+
165
+ def check_words_present(text: str, check_words: List[str]) -> float:
166
+ """Check what fraction of expected words appear in the output."""
167
+ text = text.lower()
168
+ found = sum(1 for w in check_words if w.lower() in text)
169
+ return found / len(check_words)
170
+
171
+
172
+ def llm_judge(question: str, response: str) -> Dict[str, Any]:
173
+ """Use Groq LLM to evaluate chat response quality."""
174
+ from src.utils.llm import get_fast_llm
175
+ from langchain_core.output_parsers import StrOutputParser
176
+ from langchain_core.prompts import PromptTemplate
177
+ prompt = PromptTemplate(
178
+ input_variables=["question", "response"],
179
+ template="""Evaluate this AI response. Return ONLY a JSON object, no extra text.
180
+ Question: {question}
181
+ Response: {response}
182
+
183
+ Return exactly this JSON format:
184
+ {{"relevance": <1-10>, "accuracy": <1-10>, "clarity": <1-10>, "overall": <1-10>, "comment": "<one sentence>"}}"""
185
+ )
186
+ try:
187
+ llm = get_fast_llm()
188
+ chain = prompt | llm | StrOutputParser()
189
+ result = chain.invoke({"question": question, "response": response})
190
+ # Extract JSON from response
191
+ start = result.find("{")
192
+ end = result.rfind("}") + 1
193
+ if start != -1 and end > start:
194
+ return json.loads(result[start:end])
195
+
196
+ except Exception as e:
197
+ pass
198
+
199
+ return {"relevance": 0, "accuracy": 0, "clarity": 0, "overall": 0, "comment": "Error occurred while evaluating the response."}
200
+
201
+ def print_header(title: str):
202
+ print(f"\n{'='*60}")
203
+ print(f" {title}")
204
+ print(f"{'='*60}")
205
+
206
+ def print_result(label: str, score: float, details: str = ""):
207
+ bar = "β–ˆ" * int(score * 20) + "β–‘" * (20 - int(score * 20))
208
+ print(f" {label:<25} [{bar}] {score:.2%} {details}")
209
+
210
+ # ── EVALUATION FUNCTIONS ─────────────────────────────────────────────────────
211
+
212
+ def evaluate_translation() -> Dict:
213
+ print_header("🌐 TRANSLATION EVALUATION")
214
+ scores = []
215
+
216
+ for i, test in enumerate(TRANSLATION_TESTS, 1):
217
+ print(f"\n Test {i}: English β†’ {test['target']}")
218
+ print(f" Input: {test['text'][:60]}...")
219
+
220
+ t0 = time.time()
221
+ result = translate_text(test["text"], test["source"], test["target"])
222
+ latency = time.time() - t0
223
+
224
+ score = simple_bleu(result, test["reference"])
225
+ scores.append(score)
226
+
227
+ print(f" Output: {result[:80]}...")
228
+ print(f" Reference: {test['reference'][:80]}...")
229
+ print_result("Word Overlap Score", score, f"| {latency:.2f}s")
230
+
231
+ avg = sum(scores) / len(scores)
232
+ print(f"\n βœ… Average Score: {avg:.2%}")
233
+ return {"feature": "Translation", "avg_score": avg, "scores": scores}
234
+
235
+ def evaluate_summarization() -> Dict:
236
+ print_header("πŸ“ SUMMARIZATION EVALUATION")
237
+ scores = []
238
+
239
+ for i, test in enumerate(SUMMARIZATION_TESTS, 1):
240
+ print(f"\n Test {i}: Document ({len(test['text'].split())} words)")
241
+
242
+ t0 = time.time()
243
+ result = summarize_text(test["text"], "concise")
244
+ latency = time.time() - t0
245
+
246
+ score = rouge_L(result, test["reference"])
247
+ scores.append(score)
248
+
249
+ print(f" Summary: {result[:100]}...")
250
+ print(f" Reference: {test['reference'][:100]}...")
251
+ print_result("ROUGE-L Score", score, f"| {latency:.2f}s")
252
+
253
+ avg = sum(scores) / len(scores)
254
+ print(f"\n βœ… Average Score: {avg:.2%}")
255
+ return {"feature": "Summarization", "avg_score": avg, "scores": scores}
256
+
257
+ def evaluate_keywords() -> Dict:
258
+ print_header("πŸ” KEYWORD EXTRACTION EVALUATION")
259
+ scores = []
260
+
261
+ for i, test in enumerate(KEYWORD_TESTS, 1):
262
+ print(f"\n Test {i}: ({len(test['text'].split())} words)")
263
+
264
+ t0 = time.time()
265
+ result = extract_keywords(test["text"], num_keywords=10)
266
+ latency = time.time() - t0
267
+
268
+ score = keyword_precision(result, test["expected_keywords"])
269
+ scores.append(score)
270
+
271
+ print(f" Output (first 150 chars): {result[:150]}...")
272
+ print_result("Keyword Precision", score, f"| {latency:.2f}s")
273
+
274
+ avg = sum(scores) / len(scores)
275
+ print(f"\n βœ… Average Score: {avg:.2%}")
276
+ return {"feature": "Keywords", "avg_score": avg, "scores": scores}
277
+
278
+
279
+ def evaluate_email_writer() -> Dict:
280
+ print_header("βœ‰οΈ EMAIL WRITER EVALUATION")
281
+ scores = []
282
+
283
+ # Email tests
284
+ print("\n πŸ“§ Email Generation:")
285
+ for i, test in enumerate(EMAIL_TESTS, 1):
286
+ t0 = time.time()
287
+ result = write_email(
288
+ test["type"], test["context"],
289
+ test["tone"], test["recipient"], test["sender"]
290
+ )
291
+ latency = time.time() - t0
292
+
293
+ score = check_words_present(result, test["check_words"])
294
+ scores.append(score)
295
+ print(f"\n Test {i} ({test['type']}):")
296
+ print(f" Output: {result[:120]}...")
297
+ print_result("Content Check", score, f"| {latency:.2f}s")
298
+
299
+ # Cover letter test
300
+ print("\n πŸ“„ Cover Letter Generation:")
301
+ for test in COVER_LETTER_TESTS:
302
+ t0 = time.time()
303
+ result = write_cover_letter(
304
+ test["job"], test["company"],
305
+ test["skills"], test["experience"], test["name"]
306
+ )
307
+ latency = time.time() - t0
308
+
309
+ score = check_words_present(result, test["check_words"])
310
+ scores.append(score)
311
+ print(f" Output: {result[:120]}...")
312
+ print_result("Content Check", score, f"| {latency:.2f}s")
313
+
314
+ avg = sum(scores) / len(scores)
315
+ print(f"\n βœ… Average Score: {avg:.2%}")
316
+ return {"feature": "Email Writer", "avg_score": avg, "scores": scores}
317
+
318
+
319
+ def evaluate_chat() -> Dict:
320
+ print_header("πŸ’¬ CHAT ASSISTANT EVALUATION")
321
+ session = ChatSession()
322
+ scores = []
323
+ judge_scores = []
324
+
325
+ for i, test in enumerate(CHAT_TESTS, 1):
326
+ print(f"\n Test {i}: {test['input']}")
327
+
328
+ t0 = time.time()
329
+ response = session.chat(test["input"])
330
+ latency = time.time() - t0
331
+
332
+ # Word check score
333
+ word_score = check_words_present(response, test["check"])
334
+ scores.append(word_score)
335
+
336
+ # LLM Judge score
337
+ print(f" Response: {response[:120]}...")
338
+ print_result("Word Check Score", word_score, f"| {latency:.2f}s")
339
+
340
+ print(f" πŸ€– Running LLM Judge...")
341
+ judge = llm_judge(test["input"], response)
342
+ overall = judge.get("overall", 0) / 10
343
+ judge_scores.append(overall)
344
+ print(f" LLM Judge β†’ Relevance: {judge.get('relevance')}/10 | "
345
+ f"Accuracy: {judge.get('accuracy')}/10 | "
346
+ f"Clarity: {judge.get('clarity')}/10 | "
347
+ f"Overall: {judge.get('overall')}/10")
348
+ print(f" Comment: {judge.get('comment', '')}")
349
+
350
+ avg_word = sum(scores) / len(scores)
351
+ avg_judge = sum(judge_scores) / len(judge_scores) if judge_scores else 0
352
+
353
+ print(f"\n βœ… Average Word Check: {avg_word:.2%}")
354
+ print(f" βœ… Average LLM Judge: {avg_judge:.2%}")
355
+ return {
356
+ "feature": "Chat",
357
+ "avg_word_score": avg_word,
358
+ "avg_judge_score": avg_judge,
359
+ "scores": scores
360
+ }
361
+
362
+ def evaluate_pdf_qa() -> Dict:
363
+ """PDF Q&A evaluation using a sample generated PDF."""
364
+ print_header("πŸ“„ PDF Q&A EVALUATION")
365
+
366
+ # Create a sample text file to simulate PDF content
367
+ sample_text = """
368
+ LangGraph is a library for building stateful, multi-actor applications with LLMs.
369
+ It extends LangChain by adding support for cyclic graphs, which are essential for
370
+ creating agent runtimes. LangGraph allows developers to define flows that involve
371
+ cycles, enabling more complex agent behaviors.
372
+
373
+ Key features of LangGraph include:
374
+ 1. Stateful graphs with persistent memory
375
+ 2. Support for multi-agent architectures
376
+ 3. Built-in streaming support
377
+ 4. Human-in-the-loop capabilities
378
+ 5. Time travel and debugging features
379
+
380
+ LangGraph is particularly useful for building chatbots, agents, and other
381
+ applications that require maintaining state across multiple interactions.
382
+ """
383
+
384
+ qa_tests = [
385
+ {"question": "What is LangGraph?", "expected_words": ["library", "stateful", "LLM", "graph"]},
386
+ {"question": "What are the key features of LangGraph?", "expected_words": ["stateful", "streaming", "memory", "agent"]},
387
+ {"question": "What is LangGraph useful for?", "expected_words": ["chatbot", "agent", "state"]},
388
+ ]
389
+
390
+ scores = []
391
+
392
+ try:
393
+ from src.features.pdf_qa import build_pdf_qa_graph
394
+
395
+ for i, test in enumerate(qa_tests, 1):
396
+ print(f"\n Test {i}: {test['question']}")
397
+
398
+ t0 = time.time()
399
+ graph = build_pdf_qa_graph()
400
+ result = graph.invoke({
401
+ "pdf_text": sample_text,
402
+ "question": test["question"],
403
+ "context": "",
404
+ "answer": ""
405
+ })
406
+ latency = time.time() - t0
407
+
408
+ answer = result.get("answer", "")
409
+ score = check_words_present(answer, test["expected_words"])
410
+ scores.append(score)
411
+
412
+ print(f" Answer: {answer[:120]}...")
413
+ print_result("Relevance Score", score, f"| {latency:.2f}s")
414
+
415
+ except Exception as e:
416
+ print(f" ⚠️ PDF Q&A evaluation skipped: {e}")
417
+ return {"feature": "PDF Q&A", "avg_score": 0, "scores": []}
418
+
419
+ avg = sum(scores) / len(scores)
420
+ print(f"\n βœ… Average Score: {avg:.2%}")
421
+ return {"feature": "PDF Q&A", "avg_score": avg, "scores": scores}
422
+
423
+
424
+ # ── MAIN RUNNER ──────────────────────────────────────────────────────────────
425
+
426
+ def run_all_evaluations():
427
+ print("\n" + "πŸ”¬ " * 20)
428
+ print(" GENAI TOOLKIT β€” FULL EVALUATION REPORT")
429
+ print("πŸ”¬ " * 20)
430
+
431
+ results = []
432
+ start_time = time.time()
433
+
434
+ # Run all evaluations
435
+ results.append(evaluate_translation())
436
+ results.append(evaluate_summarization())
437
+ results.append(evaluate_keywords())
438
+ results.append(evaluate_email_writer())
439
+ results.append(evaluate_chat())
440
+ results.append(evaluate_pdf_qa())
441
+
442
+ total_time = time.time() - start_time
443
+
444
+ # Final summary
445
+ print_header("πŸ“Š FINAL SUMMARY")
446
+ print()
447
+
448
+ overall_scores = []
449
+ for r in results:
450
+ score = r.get("avg_score") or r.get("avg_judge_score", 0)
451
+ overall_scores.append(score)
452
+ print_result(r["feature"], score)
453
+
454
+ overall = sum(overall_scores) / len(overall_scores)
455
+ print(f"\n {'─'*55}")
456
+ print_result("OVERALL TOOLKIT SCORE", overall)
457
+ print(f"\n ⏱️ Total evaluation time: {total_time:.1f}s")
458
+
459
+ # Save results to JSON
460
+ output = {
461
+ "overall_score": overall,
462
+ "total_time_seconds": total_time,
463
+ "features": results
464
+ }
465
+
466
+ os.makedirs("artifacts", exist_ok=True)
467
+ with open("artifacts/evaluation_results.json", "w") as f:
468
+ json.dump(output, f, indent=2)
469
+
470
+ print(f"\n πŸ’Ύ Results saved to artifacts/evaluation_results.json")
471
+ print("\n" + "βœ… " * 20 + "\n")
472
+
473
+ return output
474
+
475
+
476
+ if __name__ == "__main__":
477
+ run_all_evaluations()
478
+
479
+
480
+
481
+