improve-eval / elo.py
mdkrasnow
Add logging and client init
b2e2264
Raw History Blame Contribute Delete
3.93 kB
import os
import random
from typing import Dict, List, Any, Tuple, Optional
# Import data_utils for Argilla integration
import data_utils
def calculate_elo_update(rating_a, rating_b, a_wins):
"""Calculate updated ELO ratings after a match"""
k_factor = 32 # Determines how dramatically ratings change
expected_a = 1 / (1 + 10 ** ((rating_b - rating_a) / 400))
if a_wins:
score_a = 1
else:
score_a = 0
# Update ratings
rating_change = k_factor * (score_a - expected_a)
new_rating_a = rating_a + rating_change
new_rating_b = rating_b - rating_change
return new_rating_a, new_rating_b
def update_ratings(output_a_id, output_b_id, a_wins):
"""Update ratings for two outputs after a comparison"""
# Get current ratings from Argilla
rating_a = data_utils.get_or_update_elo_rating(output_a_id, initial_rating=1500.0)
rating_b = data_utils.get_or_update_elo_rating(output_b_id, initial_rating=1500.0)
if rating_a is None or rating_b is None:
return None, None
# Calculate new ratings
new_rating_a, new_rating_b = calculate_elo_update(rating_a, rating_b, a_wins)
# Update ratings in Argilla
data_utils.get_or_update_elo_rating(output_a_id, new_rating=new_rating_a)
data_utils.get_or_update_elo_rating(output_b_id, new_rating=new_rating_b)
return new_rating_a, new_rating_b
def select_elo_pair(outputs):
"""Select two outputs for comparison based on closest ELO ratings"""
# Get outputs with ratings
rated_outputs = []
for output in outputs:
rating = data_utils.get_or_update_elo_rating(output["id"])
if rating is not None:
output["elo_rating"] = rating
rated_outputs.append(output)
# If we have fewer than 2 rated outputs, we can't do a comparison
if len(rated_outputs) < 2:
return None, None
# Sort by ratings
rated_outputs.sort(key=lambda x: x["elo_rating"])
# Create pairs of adjacent outputs for closest ratings
pairs = []
for i in range(len(rated_outputs) - 1):
pairs.append((rated_outputs[i], rated_outputs[i+1]))
# Choose a random pair
if pairs:
return random.choice(pairs)
return None, None
def get_leaderboard(top_n=10):
"""Get the top N outputs by ELO rating from Argilla"""
try:
import argilla as rg
client = rg.Argilla(api_url=os.getenv("ARGILLA_API_URL", "http://localhost:6900"),
api_key=os.getenv("ARGILLA_API_KEY", "admin.apikey"))
print(f"[DEBUG] Getting leaderboard with top_n={top_n}")
dataset = client.datasets("danielson_outputs")
print(f"[DEBUG] Retrieved dataset for leaderboard")
# Get all records as a list
records_iterator = dataset.records
print(f"[DEBUG] Got records iterator for leaderboard")
# Filter records with ELO ratings
rated_records = []
record_count = 0
rated_count = 0
for record in records_iterator:
record_count += 1
if hasattr(record, 'responses') and record.responses and "elo_rating" in record.responses:
rated_records.append((
record.metadata["output_id"],
float(record.responses["elo_rating"])
))
rated_count += 1
print(f"[DEBUG] Found {rated_count} rated records out of {record_count} total records")
# Sort by rating (descending)
sorted_records = sorted(rated_records, key=lambda x: x[1], reverse=True)
# Return top N
return sorted_records[:top_n]
except Exception as e:
print(f"[ERROR] Error getting leaderboard from Argilla: {str(e)}")
import traceback
print(f"[ERROR] Traceback: {traceback.format_exc()}")
return []