import os import random from typing import Dict, List, Any, Tuple, Optional # Import data_utils for Argilla integration import data_utils def calculate_elo_update(rating_a, rating_b, a_wins): """Calculate updated ELO ratings after a match""" k_factor = 32 # Determines how dramatically ratings change expected_a = 1 / (1 + 10 ** ((rating_b - rating_a) / 400)) if a_wins: score_a = 1 else: score_a = 0 # Update ratings rating_change = k_factor * (score_a - expected_a) new_rating_a = rating_a + rating_change new_rating_b = rating_b - rating_change return new_rating_a, new_rating_b def update_ratings(output_a_id, output_b_id, a_wins): """Update ratings for two outputs after a comparison""" # Get current ratings from Argilla rating_a = data_utils.get_or_update_elo_rating(output_a_id, initial_rating=1500.0) rating_b = data_utils.get_or_update_elo_rating(output_b_id, initial_rating=1500.0) if rating_a is None or rating_b is None: return None, None # Calculate new ratings new_rating_a, new_rating_b = calculate_elo_update(rating_a, rating_b, a_wins) # Update ratings in Argilla data_utils.get_or_update_elo_rating(output_a_id, new_rating=new_rating_a) data_utils.get_or_update_elo_rating(output_b_id, new_rating=new_rating_b) return new_rating_a, new_rating_b def select_elo_pair(outputs): """Select two outputs for comparison based on closest ELO ratings""" # Get outputs with ratings rated_outputs = [] for output in outputs: rating = data_utils.get_or_update_elo_rating(output["id"]) if rating is not None: output["elo_rating"] = rating rated_outputs.append(output) # If we have fewer than 2 rated outputs, we can't do a comparison if len(rated_outputs) < 2: return None, None # Sort by ratings rated_outputs.sort(key=lambda x: x["elo_rating"]) # Create pairs of adjacent outputs for closest ratings pairs = [] for i in range(len(rated_outputs) - 1): pairs.append((rated_outputs[i], rated_outputs[i+1])) # Choose a random pair if pairs: return random.choice(pairs) return None, None def get_leaderboard(top_n=10): """Get the top N outputs by ELO rating from Argilla""" try: import argilla as rg client = rg.Argilla(api_url=os.getenv("ARGILLA_API_URL", "http://localhost:6900"), api_key=os.getenv("ARGILLA_API_KEY", "admin.apikey")) print(f"[DEBUG] Getting leaderboard with top_n={top_n}") dataset = client.datasets("danielson_outputs") print(f"[DEBUG] Retrieved dataset for leaderboard") # Get all records as a list records_iterator = dataset.records print(f"[DEBUG] Got records iterator for leaderboard") # Filter records with ELO ratings rated_records = [] record_count = 0 rated_count = 0 for record in records_iterator: record_count += 1 if hasattr(record, 'responses') and record.responses and "elo_rating" in record.responses: rated_records.append(( record.metadata["output_id"], float(record.responses["elo_rating"]) )) rated_count += 1 print(f"[DEBUG] Found {rated_count} rated records out of {record_count} total records") # Sort by rating (descending) sorted_records = sorted(rated_records, key=lambda x: x[1], reverse=True) # Return top N return sorted_records[:top_n] except Exception as e: print(f"[ERROR] Error getting leaderboard from Argilla: {str(e)}") import traceback print(f"[ERROR] Traceback: {traceback.format_exc()}") return []