Spaces:
Sleeping
Sleeping
File size: 9,486 Bytes
897e2b2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 | """
Tests for RecommendationExplainer LLM component.
These tests verify that the LLM-powered explanation generator:
1. Produces correctly formatted output with expected structure
2. Generates content that matches user requirements
3. Handles different input scenarios consistently
"""
import os
import sys
import pytest
from dotenv import load_dotenv
# Add src to path
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
load_dotenv()
from recommendation_explainer import RecommendationExplainer
# Sample test data
SAMPLE_RECOMMENDATIONS = [
{
'model': 'GPT-4',
'organization': 'OpenAI',
'arena_score': 1250,
'votes': 50000,
'license': 'Proprietary',
'knowledge_cutoff': '2023/09'
},
{
'model': 'Claude-3-Opus',
'organization': 'Anthropic',
'arena_score': 1240,
'votes': 45000,
'license': 'Proprietary',
'knowledge_cutoff': '2023/08'
},
{
'model': 'Gemini-Pro',
'organization': 'Google',
'arena_score': 1230,
'votes': 40000,
'license': 'Proprietary',
'knowledge_cutoff': '2023/11'
}
]
class TestRecommendationExplainerOutput:
"""Test that LLM output has the correct structure and format."""
@pytest.fixture
def explainer(self):
"""Create an explainer instance for testing."""
return RecommendationExplainer()
def test_explanation_is_non_empty_string(self, explainer):
"""Test that the LLM returns a non-empty string response."""
explanation = explainer.generate_explanation(
use_case='conversational_knowledge',
recommendations=SAMPLE_RECOMMENDATIONS,
model_type='no_preference'
)
assert isinstance(explanation, str), "Explanation should be a string"
assert len(explanation) > 0, "Explanation should not be empty"
assert len(explanation) > 100, "Explanation should be substantial (>100 chars)"
def test_explanation_has_paragraph_structure(self, explainer):
"""Test that the explanation has multiple paragraphs as expected."""
explanation = explainer.generate_explanation(
use_case='technical_developer',
recommendations=SAMPLE_RECOMMENDATIONS,
model_type='proprietary_only_enterprise'
)
# Check that explanation has multiple paragraphs (separated by double newlines)
paragraphs = [p.strip() for p in explanation.split('\n\n') if p.strip()]
assert len(paragraphs) >= 2, f"Explanation should have at least 2 paragraphs, got {len(paragraphs)}"
# Each paragraph should be substantial
for i, para in enumerate(paragraphs):
assert len(para) > 50, f"Paragraph {i+1} should be substantial (>50 chars), got {len(para)}"
def test_explanation_mentions_model_names(self, explainer):
"""Test that the LLM output mentions the recommended model names."""
explanation = explainer.generate_explanation(
use_case='creative_content',
recommendations=SAMPLE_RECOMMENDATIONS,
model_type='no_preference'
)
# Check that at least 2 of the 3 model names are mentioned
model_names = ['GPT-4', 'Claude', 'Gemini']
mentions = sum(1 for name in model_names if name.lower() in explanation.lower())
assert mentions >= 2, f"Explanation should mention at least 2 model names, found {mentions}"
class TestRecommendationExplainerConsistency:
"""Test that prompt variations produce consistent and appropriate results."""
@pytest.fixture
def explainer(self):
"""Create an explainer instance for testing."""
return RecommendationExplainer()
def test_visual_ai_context_included(self, explainer):
"""Test that visual AI type is reflected in the explanation."""
explanation = explainer.generate_explanation(
use_case='visual_ai',
recommendations=SAMPLE_RECOMMENDATIONS,
visual_ai_type='image_generation',
model_type='no_preference'
)
# Check that visual/image-related terms appear in the explanation
visual_keywords = ['visual', 'image', 'vision', 'picture', 'photo', 'generation']
has_visual_context = any(keyword in explanation.lower() for keyword in visual_keywords)
assert has_visual_context, "Explanation for visual AI should mention visual/image-related concepts"
def test_open_source_preference_reflected(self, explainer):
"""Test that model type preference is reflected in explanation."""
# Create open-source recommendations
open_source_recs = [
{
'model': 'Llama-3-70B',
'organization': 'Meta',
'arena_score': 1200,
'votes': 35000,
'license': 'Apache 2.0',
'knowledge_cutoff': '2024/03'
},
{
'model': 'Mixtral-8x7B',
'organization': 'Mistral',
'arena_score': 1190,
'votes': 30000,
'license': 'Apache 2.0',
'knowledge_cutoff': '2024/01'
},
{
'model': 'Qwen-2-72B',
'organization': 'Alibaba',
'arena_score': 1180,
'votes': 25000,
'license': 'Apache 2.0',
'knowledge_cutoff': '2024/02'
}
]
explanation = explainer.generate_explanation(
use_case='technical_developer',
recommendations=open_source_recs,
model_type='open_only'
)
# Check for open-source related keywords
open_keywords = ['open', 'source', 'community', 'accessible', 'weights', 'transparent']
has_open_context = any(keyword in explanation.lower() for keyword in open_keywords)
assert has_open_context, "Explanation should reflect open-source preference"
def test_different_use_cases_produce_different_explanations(self, explainer):
"""Test that different use cases generate contextually different explanations."""
explanation_conversational = explainer.generate_explanation(
use_case='conversational_knowledge',
recommendations=SAMPLE_RECOMMENDATIONS
)
explanation_technical = explainer.generate_explanation(
use_case='technical_developer',
recommendations=SAMPLE_RECOMMENDATIONS
)
# The explanations should be different (not identical)
assert explanation_conversational != explanation_technical, \
"Different use cases should produce different explanations"
# Conversational might mention chat/conversation/assistant
conversational_keywords = ['chat', 'conversation', 'assistant', 'dialogue', 'interact']
has_conversational = any(kw in explanation_conversational.lower() for kw in conversational_keywords)
# Technical might mention code/development/programming
technical_keywords = ['code', 'develop', 'program', 'technical', 'engineer', 'software']
has_technical = any(kw in explanation_technical.lower() for kw in technical_keywords)
# At least one should match its context
assert has_conversational or has_technical, \
"Explanations should include use-case-specific terminology"
class TestRecommendationExplainerPromptProcessing:
"""Test that the prompt is built and processed correctly."""
@pytest.fixture
def explainer(self):
"""Create an explainer instance for testing."""
return RecommendationExplainer()
def test_prompt_includes_all_recommendations(self, explainer):
"""Test that the internal prompt includes all 3 recommendations."""
prompt = explainer._build_prompt(
use_case='productivity_information',
recommendations=SAMPLE_RECOMMENDATIONS,
model_type='no_preference'
)
# Check that all model names appear in the prompt
for rec in SAMPLE_RECOMMENDATIONS:
assert rec['model'] in prompt, f"Model {rec['model']} should be in prompt"
assert rec['organization'] in prompt, f"Organization {rec['organization']} should be in prompt"
def test_prompt_formats_use_case_correctly(self, explainer):
"""Test that use case is formatted properly in the prompt."""
prompt = explainer._build_prompt(
use_case='advanced_automation',
recommendations=SAMPLE_RECOMMENDATIONS
)
# Use case should be formatted (underscores replaced with spaces, title case)
assert 'Advanced Automation' in prompt, "Use case should be formatted in title case"
def test_fallback_explanation_works(self, explainer):
"""Test that fallback explanation is provided when API fails."""
fallback = explainer._get_fallback_explanation(
use_case='conversational_knowledge',
recommendations=SAMPLE_RECOMMENDATIONS
)
assert isinstance(fallback, str), "Fallback should be a string"
assert len(fallback) > 100, "Fallback should be substantial"
assert 'LMSYS' in fallback or 'Arena' in fallback, \
"Fallback should mention the leaderboard source"
if __name__ == "__main__":
# Run tests with pytest
pytest.main([__file__, '-v'])
|