Spaces:
Sleeping
Sleeping
File size: 9,907 Bytes
0684049 ed724f1 1678ee5 0684049 1678ee5 0684049 1678ee5 0684049 897e2b2 0684049 897e2b2 0684049 897e2b2 0684049 ed724f1 0684049 1678ee5 0684049 ed724f1 0684049 ed724f1 1678ee5 0684049 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 | import os
import pandas as pd
from typing import Dict, List, Optional, Tuple
class RecommendationEngine:
def __init__(self):
"""Initialize the recommendation engine with data directory path."""
self.data_dir = os.path.join(os.path.dirname(__file__), "..", "data")
# Organization to logo filename mapping
self.org_to_logo = {
'OpenAI': 'openai.png',
'Google': 'google.png',
'Anthropic': 'anthropic.png',
'Meta': 'meta.png',
'Microsoft': 'microsoft.png',
'Mistral': 'mistral.png',
'Cohere': 'cohere.png',
'xAI': 'xai.png',
'DeepSeek': 'deepseek.png',
'Alibaba': 'alibaba.png',
'Z.ai': 'zai.png',
'Moonshot': 'moonshot.png',
'Tencent': 'tencent.png',
'LG AI Research': 'lg.png',
'Perplexity': 'perplexity.png',
'AI21 Labs': 'ai21.png',
'Reka': 'reka.png',
'Inflection': 'inflection.png',
'Stability AI': 'stability.png',
}
# Mapping from questionnaire choices to CSV files (with pricing data)
self.use_case_mapping = {
'conversational_knowledge': ['lmarena_text_with_pricing_or.csv'],
'productivity_information': ['lmarena_text_with_pricing_or.csv'],
'creative_content': ['lmarena_text_with_pricing_or.csv'],
'technical_developer': ['lmarena_webdev_with_pricing_or.csv'],
'advanced_automation': ['lmarena_text_with_pricing_or.csv'],
'visual_ai': ['lmarena_vision_with_pricing_or.csv', 'lmarena_image_with_pricing.csv', 'lmarena_image-edit_with_pricing.csv']
}
# Visual AI sub-category mapping
self.visual_ai_mapping = {
'image_understanding': ['lmarena_vision_with_pricing_or.csv'],
'image_generation': ['lmarena_image_with_pricing.csv'],
'image_editing': ['lmarena_image-edit_with_pricing.csv']
}
def get_relevant_csv_files(self, use_case: str, visual_ai_type: Optional[str] = None) -> List[str]:
"""
Map user's use case choice to relevant CSV files.
Args:
use_case: Primary use case selected by user
visual_ai_type: Visual AI type if use_case is 'visual_ai'
Returns:
List of CSV filenames to use for recommendations
"""
if use_case == 'visual_ai' and visual_ai_type:
# If Visual AI is selected, use the specific visual AI mapping
return self.visual_ai_mapping.get(visual_ai_type, [])
else:
# For all other use cases, use the primary mapping
return self.use_case_mapping.get(use_case, [])
def load_leaderboard_data(self, csv_files: List[str]) -> Dict[str, pd.DataFrame]:
"""
Load leaderboard data from specified CSV files.
Args:
csv_files: List of CSV filenames to load
Returns:
Dictionary mapping filename to DataFrame
"""
data = {}
for csv_file in csv_files:
file_path = os.path.join(self.data_dir, csv_file)
if os.path.exists(file_path):
try:
df = pd.read_csv(file_path)
data[csv_file] = df
print(f"β
Loaded {csv_file}: {len(df)} models")
except Exception as e:
print(f"β Error loading {csv_file}: {e}")
else:
print(f"β οΈ File not found: {csv_file}")
return data
def get_top_recommendations(self, use_case: str, visual_ai_type: Optional[str] = None, model_type: Optional[str] = None, priority: Optional[str] = None, top_n: int = 3) -> List[Dict]:
"""
Get top N model recommendations for a given use case.
Args:
use_case: Primary use case selected by user
visual_ai_type: Visual AI type if use_case is 'visual_ai'
model_type: Model type preference ('open_only', 'proprietary_only_enterprise', 'no_preference')
priority: User priority ('lower_cost' or 'better_performance')
top_n: Number of top recommendations to return (default: 3)
Returns:
List of dictionaries containing model information
"""
# Get relevant CSV files
csv_files = self.get_relevant_csv_files(use_case, visual_ai_type)
if not csv_files:
print(f"β No CSV files mapped for use case: {use_case}")
return []
# Load the data
loaded_data = self.load_leaderboard_data(csv_files)
if not loaded_data:
print(f"β No data loaded for use case: {use_case}")
return []
# Get the first CSV file's data (primary ranking source)
primary_csv = csv_files[0]
df = loaded_data.get(primary_csv)
if df is None or df.empty:
print(f"β No data in primary CSV: {primary_csv}")
return []
# Apply model type filter based on user preference
if model_type == 'open_only':
# Filter to exclude proprietary models
df = df[df['license'] != 'Proprietary']
print(f"π Filtered to open weights models: {len(df)} models")
elif model_type == 'proprietary_only_enterprise':
# Filter to include only proprietary models
df = df[df['license'] == 'Proprietary']
print(f"π Filtered to proprietary models: {len(df)} models")
# If 'no_preference', no filtering is applied
if df.empty:
print(f"β No models found after applying model_type filter: {model_type}")
return []
# Apply sorting based on user's priority
if priority == 'lower_cost':
# Sort by pricing (ascending) - lower cost first
# Convert pricing to float, handling non-numeric values
df['pricing_numeric'] = pd.to_numeric(df['pricing'], errors='coerce')
df = df.sort_values('pricing_numeric', ascending=True)
print(f"π° Sorted by cost (lowest first)")
else:
# Default: Sort by rank (better performance first)
# CSV is already sorted by rank, but ensure it
df = df.sort_values('rank', ascending=True)
print(f"π Sorted by performance (rank)")
# Get top N models
top_models = df.head(top_n)
# Convert to list of dictionaries with relevant fields
recommendations = []
for idx, row in top_models.iterrows():
organization = row.get('organization', 'Unknown')
# Get logo filename from mapping
logo_filename = self.org_to_logo.get(organization, '')
logo_url = f'/static/images/logos/{logo_filename}' if logo_filename else ''
model_info = {
'rank': int(row.get('rank', idx + 1)),
'model': row.get('displayed_name', row.get('model', 'Unknown')), # Use displayed_name if available
'arena_score': row.get('arena_score', 'N/A'),
'votes': row.get('votes', 'N/A'),
'organization': organization,
'license': row.get('license', 'Unknown'),
'url': row.get('url', ''),
'ci': row.get('95_pct_ci', 'N/A'),
'logo_url': logo_url,
'knowledge_cutoff': row.get('knowledge_cutoff', 'N/A'),
'pricing': row.get('pricing', 'N/A') # Add pricing information
}
recommendations.append(model_info)
print(f"β
Generated {len(recommendations)} recommendations")
return recommendations
def process_user_input(self, questionnaire_data: Dict) -> Tuple[List[str], Dict[str, pd.DataFrame]]:
"""
Process user questionnaire input and return relevant data.
Args:
questionnaire_data: Dictionary containing user's questionnaire responses
Returns:
Tuple of (csv_files_used, loaded_dataframes)
"""
# Extract primary use case
use_case = questionnaire_data.get('use_case')
visual_ai_type = questionnaire_data.get('visual_ai_type')
print(f"π― User selected use case: {use_case}")
if visual_ai_type:
print(f"πΌοΈ Visual AI type: {visual_ai_type}")
# Get relevant CSV files
csv_files = self.get_relevant_csv_files(use_case, visual_ai_type)
if not csv_files:
print(f"β No CSV files mapped for use case: {use_case}")
return [], {}
print(f"π Mapped to CSV files: {csv_files}")
# Load the data
loaded_data = self.load_leaderboard_data(csv_files)
return csv_files, loaded_data
def test_mapping(self):
"""Test the mapping logic with sample inputs."""
print("π§ͺ Testing Use Case Mapping:")
print("=" * 50)
test_cases = [
{'use_case': 'conversational_knowledge'},
{'use_case': 'productivity_information'},
{'use_case': 'creative_content'},
{'use_case': 'technical_developer'},
{'use_case': 'advanced_automation'},
{'use_case': 'visual_ai', 'visual_ai_type': 'image_understanding'},
{'use_case': 'visual_ai', 'visual_ai_type': 'image_generation'},
{'use_case': 'visual_ai', 'visual_ai_type': 'image_editing'},
]
for test_case in test_cases:
print(f"\nTest case: {test_case}")
csv_files, data = self.process_user_input(test_case)
print(f"Result: {len(data)} datasets loaded")
print("-" * 30)
if __name__ == "__main__":
# Initialize and test the recommendation engine
engine = RecommendationEngine()
engine.test_mapping() |