Spaces:
Sleeping
Sleeping
Download elo.py from mattkrasnow/improve-eval: direct link, hf CLI and curl.
- Browser
- Download file 3.93 kB
-
https://huggingface.co/spaces/mattkrasnow/improve-eval/resolve/main/elo.py
- Command line
-
hf download hf://spaces/mattkrasnow/improve-eval/elo.py
-
curl -L -o elo.py https://huggingface.co/spaces/mattkrasnow/improve-eval/resolve/main/elo.py
3.93 kB
| import os | |
| import random | |
| from typing import Dict, List, Any, Tuple, Optional | |
| # Import data_utils for Argilla integration | |
| import data_utils | |
| def calculate_elo_update(rating_a, rating_b, a_wins): | |
| """Calculate updated ELO ratings after a match""" | |
| k_factor = 32 # Determines how dramatically ratings change | |
| expected_a = 1 / (1 + 10 ** ((rating_b - rating_a) / 400)) | |
| if a_wins: | |
| score_a = 1 | |
| else: | |
| score_a = 0 | |
| # Update ratings | |
| rating_change = k_factor * (score_a - expected_a) | |
| new_rating_a = rating_a + rating_change | |
| new_rating_b = rating_b - rating_change | |
| return new_rating_a, new_rating_b | |
| def update_ratings(output_a_id, output_b_id, a_wins): | |
| """Update ratings for two outputs after a comparison""" | |
| # Get current ratings from Argilla | |
| rating_a = data_utils.get_or_update_elo_rating(output_a_id, initial_rating=1500.0) | |
| rating_b = data_utils.get_or_update_elo_rating(output_b_id, initial_rating=1500.0) | |
| if rating_a is None or rating_b is None: | |
| return None, None | |
| # Calculate new ratings | |
| new_rating_a, new_rating_b = calculate_elo_update(rating_a, rating_b, a_wins) | |
| # Update ratings in Argilla | |
| data_utils.get_or_update_elo_rating(output_a_id, new_rating=new_rating_a) | |
| data_utils.get_or_update_elo_rating(output_b_id, new_rating=new_rating_b) | |
| return new_rating_a, new_rating_b | |
| def select_elo_pair(outputs): | |
| """Select two outputs for comparison based on closest ELO ratings""" | |
| # Get outputs with ratings | |
| rated_outputs = [] | |
| for output in outputs: | |
| rating = data_utils.get_or_update_elo_rating(output["id"]) | |
| if rating is not None: | |
| output["elo_rating"] = rating | |
| rated_outputs.append(output) | |
| # If we have fewer than 2 rated outputs, we can't do a comparison | |
| if len(rated_outputs) < 2: | |
| return None, None | |
| # Sort by ratings | |
| rated_outputs.sort(key=lambda x: x["elo_rating"]) | |
| # Create pairs of adjacent outputs for closest ratings | |
| pairs = [] | |
| for i in range(len(rated_outputs) - 1): | |
| pairs.append((rated_outputs[i], rated_outputs[i+1])) | |
| # Choose a random pair | |
| if pairs: | |
| return random.choice(pairs) | |
| return None, None | |
| def get_leaderboard(top_n=10): | |
| """Get the top N outputs by ELO rating from Argilla""" | |
| try: | |
| import argilla as rg | |
| client = rg.Argilla(api_url=os.getenv("ARGILLA_API_URL", "http://localhost:6900"), | |
| api_key=os.getenv("ARGILLA_API_KEY", "admin.apikey")) | |
| print(f"[DEBUG] Getting leaderboard with top_n={top_n}") | |
| dataset = client.datasets("danielson_outputs") | |
| print(f"[DEBUG] Retrieved dataset for leaderboard") | |
| # Get all records as a list | |
| records_iterator = dataset.records | |
| print(f"[DEBUG] Got records iterator for leaderboard") | |
| # Filter records with ELO ratings | |
| rated_records = [] | |
| record_count = 0 | |
| rated_count = 0 | |
| for record in records_iterator: | |
| record_count += 1 | |
| if hasattr(record, 'responses') and record.responses and "elo_rating" in record.responses: | |
| rated_records.append(( | |
| record.metadata["output_id"], | |
| float(record.responses["elo_rating"]) | |
| )) | |
| rated_count += 1 | |
| print(f"[DEBUG] Found {rated_count} rated records out of {record_count} total records") | |
| # Sort by rating (descending) | |
| sorted_records = sorted(rated_records, key=lambda x: x[1], reverse=True) | |
| # Return top N | |
| return sorted_records[:top_n] | |
| except Exception as e: | |
| print(f"[ERROR] Error getting leaderboard from Argilla: {str(e)}") | |
| import traceback | |
| print(f"[ERROR] Traceback: {traceback.format_exc()}") | |
| return [] | |