Spaces:
Sleeping
Sleeping
Download app.py from mattkrasnow/improve-eval: direct link, hf CLI and curl.
- Browser
- Download file 25.6 kB
-
https://huggingface.co/spaces/mattkrasnow/improve-eval/resolve/main/app.py
- Command line
-
hf download hf://spaces/mattkrasnow/improve-eval/app.py
-
curl -L -o app.py https://huggingface.co/spaces/mattkrasnow/improve-eval/resolve/main/app.py
25.6 kB
| import gradio as gr | |
| import random | |
| import json | |
| import uuid | |
| import datetime | |
| import os | |
| from typing import Dict, List, Any, Tuple, Optional | |
| # Import our modules | |
| import data_utils | |
| import elo | |
| # Initialize Argilla and data for first cycle | |
| data_utils.init_argilla() | |
| data_utils.initialize_cycle() | |
| def initialize_state(): | |
| """Initialize the application state""" | |
| return { | |
| "current_stage": 1, # 1 = Individual assessment, 2 = Comparison | |
| "examples_completed": 0, | |
| "current_example_index": 0, | |
| "elo_comparisons_done": 0, | |
| "elo_pair": None, | |
| "current_choice": None # Track the user's current choice (good, fine, bad) | |
| } | |
| def load_example(state): | |
| """Load the current example for display""" | |
| try: | |
| # Get the current output | |
| current_output = data_utils.all_outputs[state["current_example_index"]] | |
| # Return values in the same order as the outputs parameter in demo.load | |
| return current_output["component_id"], current_output["low_inference_notes"], current_output["output"]["summary"] | |
| except (IndexError, KeyError) as e: | |
| # Handle errors by initializing a new cycle | |
| data_utils.initialize_cycle() | |
| state["current_example_index"] = 0 | |
| current_output = data_utils.all_outputs[0] | |
| # Return values in the same order as the outputs parameter in demo.load | |
| return current_output["component_id"], current_output["low_inference_notes"], current_output["output"]["summary"] | |
| def select_choice(choice, state): | |
| """Set the choice in the state and provide visual feedback""" | |
| state["current_choice"] = choice | |
| print(f"User selected: {choice}") | |
| # Update button states | |
| good_btn_update = gr.update( | |
| variant="primary" if choice == "good" else "secondary", | |
| elem_classes=["good-button", "selected-button"] if choice == "good" else ["good-button"] | |
| ) | |
| fine_btn_update = gr.update( | |
| variant="primary" if choice == "fine" else "secondary", | |
| elem_classes=["fine-button", "selected-button"] if choice == "fine" else ["fine-button"] | |
| ) | |
| bad_btn_update = gr.update( | |
| variant="primary" if choice == "bad" else "secondary", | |
| elem_classes=["bad-button", "selected-button"] if choice == "bad" else ["bad-button"] | |
| ) | |
| log_msg_update = gr.update(value=f"Selected rating: {choice.upper()}", visible=True) | |
| # Return individual updates for each component | |
| return good_btn_update, fine_btn_update, bad_btn_update, log_msg_update, state | |
| def stage1_submit(explanation, state): | |
| """Process Stage 1 submission (individual assessment)""" | |
| # Validate input | |
| if not state["current_choice"]: | |
| error_msg = gr.update(value="Please select a rating (Good, Fine, or Bad).", visible=True) | |
| log_msg = gr.update(value="Submission failed: No rating selected", visible=True) | |
| # Return empty/unchanged values for all other outputs | |
| return ( | |
| gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), | |
| gr.update(), error_msg, log_msg, gr.update(), gr.update(), gr.update(), state | |
| ) | |
| if not explanation.strip(): | |
| error_msg = gr.update(value="Please provide an explanation for your choice.", visible=True) | |
| log_msg = gr.update(value="Submission failed: No explanation provided", visible=True) | |
| # Return empty/unchanged values for all other outputs | |
| return ( | |
| gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), | |
| gr.update(), error_msg, log_msg, gr.update(), gr.update(), gr.update(), state | |
| ) | |
| # Get current output | |
| current_output = data_utils.all_outputs[state["current_example_index"]] | |
| # Log the action | |
| print(f"Submitting feedback for output ID: {current_output['id']}") | |
| print(f"Choice: {state['current_choice']}, Explanation: {explanation}") | |
| # Reinitialize Argilla client to ensure fresh connection | |
| data_utils.init_argilla() | |
| # Store feedback in Argilla | |
| try: | |
| success = data_utils.store_feedback( | |
| output_id=current_output["id"], | |
| choice=state["current_choice"], | |
| explanation=explanation, | |
| stage="individual" | |
| ) | |
| except Exception as e: | |
| print(f"[ERROR] Exception during store_feedback: {str(e)}") | |
| import traceback | |
| print(f"[ERROR] Traceback: {traceback.format_exc()}") | |
| success = False | |
| if not success: | |
| error_msg = gr.update(value=f"Failed to store feedback in Argilla. Please check logs for details.", visible=True) | |
| log_msg = gr.update(value="Submission failed: Error storing in Argilla", visible=True) | |
| # Return empty/unchanged values for all other outputs | |
| return ( | |
| gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), | |
| gr.update(), error_msg, log_msg, gr.update(), gr.update(), gr.update(), state | |
| ) | |
| # Increment counter | |
| state["examples_completed"] += 1 | |
| # Reset current choice | |
| state["current_choice"] = None | |
| # Determine if we should move to next example or to stage 2 | |
| if state["examples_completed"] >= len(data_utils.all_outputs): | |
| # Move to stage 2 (ELO comparisons) | |
| state["current_stage"] = 2 | |
| state["elo_comparisons_done"] = 0 | |
| # Select first pair for comparison | |
| output_a, output_b = elo.select_elo_pair(data_utils.all_outputs) | |
| # Check if we have a valid pair | |
| if output_a is None or output_b is None: | |
| # Not enough rated outputs, start a new cycle | |
| data_utils.initialize_cycle() | |
| state = initialize_state() | |
| current_output = data_utils.all_outputs[state["current_example_index"]] | |
| return ( | |
| gr.update(visible=True), # stage1_group | |
| gr.update(visible=False), # stage2_group | |
| gr.update(value=current_output["component_id"]), # component_id_display | |
| gr.update(value=current_output["low_inference_notes"]), # notes_display | |
| gr.update(value=current_output["output"]["summary"]), # output_display | |
| gr.update(value=""), # explanation_input | |
| gr.update(value="", visible=False), # error_message | |
| gr.update(value="Started new cycle - not enough rated outputs for comparison", visible=True), # log_message | |
| gr.update(variant="secondary"), # good_btn | |
| gr.update(variant="secondary"), # fine_btn | |
| gr.update(variant="secondary"), # bad_btn | |
| state | |
| ) | |
| state["elo_pair"] = (output_a, output_b) | |
| return ( | |
| gr.update(visible=False), # stage1_group | |
| gr.update(visible=True), # stage2_group | |
| gr.update(), # component_id_display (unchanged) | |
| gr.update(), # notes_display (unchanged) | |
| gr.update(), # output_display (unchanged) | |
| gr.update(value=""), # explanation_input | |
| gr.update(value="", visible=False), # error_message | |
| gr.update(value="Moving to comparison stage", visible=True), # log_message | |
| gr.update(variant="secondary"), # good_btn | |
| gr.update(variant="secondary"), # fine_btn | |
| gr.update(variant="secondary"), # bad_btn | |
| state | |
| ) | |
| else: | |
| # Move to next example | |
| state["current_example_index"] = state["examples_completed"] | |
| current_output = data_utils.all_outputs[state["current_example_index"]] | |
| return ( | |
| gr.update(visible=True), # stage1_group | |
| gr.update(visible=False), # stage2_group | |
| gr.update(value=current_output["component_id"]), # component_id_display | |
| gr.update(value=current_output["low_inference_notes"]), # notes_display | |
| gr.update(value=current_output["output"]["summary"]), # output_display | |
| gr.update(value=""), # explanation_input | |
| gr.update(value="", visible=False), # error_message | |
| gr.update(value=f"Submission successful! Moving to next example ({state['examples_completed']+1}/{len(data_utils.all_outputs)})", visible=True), # log_message | |
| gr.update(variant="secondary"), # good_btn | |
| gr.update(variant="secondary"), # fine_btn | |
| gr.update(variant="secondary"), # bad_btn | |
| state | |
| ) | |
| def select_comparison_choice(choice, state): | |
| """Set the comparison choice in the state and provide visual feedback""" | |
| state["current_choice"] = choice | |
| print(f"User selected output: {choice}") | |
| # Update button states | |
| choose_a_btn_update = gr.update( | |
| variant="primary" if choice == "a" else "secondary", | |
| elem_classes=["selected-button"] if choice == "a" else [] | |
| ) | |
| choose_b_btn_update = gr.update( | |
| variant="primary" if choice == "b" else "secondary", | |
| elem_classes=["selected-button"] if choice == "b" else [] | |
| ) | |
| log_msg_update = gr.update(value=f"Selected output: {choice.upper()}", visible=True) | |
| # Return individual updates for each component | |
| return choose_a_btn_update, choose_b_btn_update, log_msg_update, state | |
| def stage2_submit(explanation, state): | |
| """Process Stage 2 submission (comparative assessment)""" | |
| # Validate input | |
| if not state["current_choice"]: | |
| error_msg = gr.update(value="Please select which output is better (A or B).", visible=True) | |
| log_msg = gr.update(value="Submission failed: No output selected", visible=True) | |
| # Return unchanged values for all other outputs | |
| return ( | |
| gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), | |
| gr.update(), gr.update(), gr.update(), gr.update(), | |
| error_msg, log_msg, gr.update(), gr.update(), state | |
| ) | |
| if not explanation.strip(): | |
| error_msg = gr.update(value="Please provide an explanation for your choice.", visible=True) | |
| log_msg = gr.update(value="Submission failed: No explanation provided", visible=True) | |
| # Return unchanged values for all other outputs | |
| return ( | |
| gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), | |
| gr.update(), gr.update(), gr.update(), gr.update(), | |
| error_msg, log_msg, gr.update(), gr.update(), state | |
| ) | |
| # Get current comparison pair | |
| output_a = state["elo_pair"][0] | |
| output_b = state["elo_pair"][1] | |
| # Determine which output was chosen | |
| a_wins = (state["current_choice"] == "a") | |
| # Log the action | |
| print(f"Comparing outputs: {output_a['id']} vs {output_b['id']}") | |
| print(f"User chose: {'A' if a_wins else 'B'}, Explanation: {explanation}") | |
| # Reinitialize Argilla client to ensure fresh connection | |
| data_utils.init_argilla() | |
| # Update ELO ratings | |
| try: | |
| new_rating_a, new_rating_b = elo.update_ratings(output_a["id"], output_b["id"], a_wins) | |
| except Exception as e: | |
| print(f"[ERROR] Exception during ELO rating update: {str(e)}") | |
| import traceback | |
| print(f"[ERROR] Traceback: {traceback.format_exc()}") | |
| new_rating_a, new_rating_b = None, None | |
| if new_rating_a is None or new_rating_b is None: | |
| error_msg = gr.update(value="Failed to update ELO ratings. Please check logs for details.", visible=True) | |
| log_msg = gr.update(value="Submission failed: Error updating ELO ratings", visible=True) | |
| # Return unchanged values for all other outputs | |
| return ( | |
| gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), | |
| gr.update(), gr.update(), gr.update(), gr.update(), | |
| error_msg, log_msg, gr.update(), gr.update(), state | |
| ) | |
| # Store feedback | |
| try: | |
| success = data_utils.store_feedback( | |
| output_id=output_a["id"] if a_wins else output_b["id"], | |
| choice=f"preferred_over_{output_b['id'] if a_wins else output_a['id']}", | |
| explanation=explanation, | |
| stage="comparison" | |
| ) | |
| except Exception as e: | |
| print(f"[ERROR] Exception during store_feedback for comparison: {str(e)}") | |
| import traceback | |
| print(f"[ERROR] Traceback: {traceback.format_exc()}") | |
| success = False | |
| if not success: | |
| error_msg = gr.update(value="Failed to store comparison feedback in Argilla. Please check logs for details.", visible=True) | |
| log_msg = gr.update(value="Submission failed: Error storing comparison in Argilla", visible=True) | |
| # Return unchanged values for all other outputs | |
| return ( | |
| gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), | |
| gr.update(), gr.update(), gr.update(), gr.update(), | |
| error_msg, log_msg, gr.update(), gr.update(), state | |
| ) | |
| # Reset current choice | |
| state["current_choice"] = None | |
| # Increment counter | |
| state["elo_comparisons_done"] += 1 | |
| # Determine if we should continue with more ELO comparisons or restart | |
| if state["elo_comparisons_done"] >= 5: # Do 5 comparisons per cycle | |
| # Restart the process with a new cycle | |
| data_utils.initialize_cycle() | |
| state = initialize_state() | |
| current_output = data_utils.all_outputs[state["current_example_index"]] | |
| return ( | |
| gr.update(visible=True), # stage1_group | |
| gr.update(visible=False), # stage2_group | |
| gr.update(value=current_output["component_id"]), # component_id_display | |
| gr.update(value=current_output["low_inference_notes"]), # notes_display | |
| gr.update(value=current_output["output"]["summary"]), # output_display | |
| gr.update(), # notes_display_elo (unchanged) | |
| gr.update(), # output_a (unchanged) | |
| gr.update(), # output_b (unchanged) | |
| gr.update(value=""), # comparison_explanation | |
| gr.update(value="", visible=False), # error_message | |
| gr.update(value="Completed comparison cycle. Starting new evaluation cycle.", visible=True), # log_message | |
| gr.update(variant="secondary"), # choose_a_btn | |
| gr.update(variant="secondary"), # choose_b_btn | |
| state | |
| ) | |
| else: | |
| # Get a new comparison pair | |
| output_a, output_b = elo.select_elo_pair(data_utils.all_outputs) | |
| if output_a is None or output_b is None: | |
| # Not enough rated outputs, start a new cycle | |
| data_utils.initialize_cycle() | |
| state = initialize_state() | |
| current_output = data_utils.all_outputs[state["current_example_index"]] | |
| return ( | |
| gr.update(visible=True), # stage1_group | |
| gr.update(visible=False), # stage2_group | |
| gr.update(value=current_output["component_id"]), # component_id_display | |
| gr.update(value=current_output["low_inference_notes"]), # notes_display | |
| gr.update(value=current_output["output"]["summary"]), # output_display | |
| gr.update(), # notes_display_elo (unchanged) | |
| gr.update(), # output_a (unchanged) | |
| gr.update(), # output_b (unchanged) | |
| gr.update(value=""), # comparison_explanation | |
| gr.update(value="", visible=False), # error_message | |
| gr.update(value="Not enough rated outputs for comparison. Starting new cycle.", visible=True), # log_message | |
| gr.update(variant="secondary"), # choose_a_btn | |
| gr.update(variant="secondary"), # choose_b_btn | |
| state | |
| ) | |
| state["elo_pair"] = (output_a, output_b) | |
| return ( | |
| gr.update(visible=False), # stage1_group | |
| gr.update(visible=True), # stage2_group | |
| gr.update(), # component_id_display (unchanged) | |
| gr.update(), # notes_display (unchanged) | |
| gr.update(), # output_display (unchanged) | |
| gr.update(value=output_a["low_inference_notes"]), # notes_display_elo | |
| gr.update(value=output_a["output"]["summary"]), # output_a | |
| gr.update(value=output_b["output"]["summary"]), # output_b | |
| gr.update(value=""), # comparison_explanation | |
| gr.update(value="", visible=False), # error_message | |
| gr.update(value=f"Comparison submitted successfully! Moving to next comparison ({state['elo_comparisons_done']+1}/5)", visible=True), # log_message | |
| gr.update(variant="secondary"), # choose_a_btn | |
| gr.update(variant="secondary"), # choose_b_btn | |
| state | |
| ) | |
| def get_leaderboard_display(): | |
| """Get the leaderboard display for top outputs""" | |
| leaderboard = elo.get_leaderboard(top_n=10) | |
| if not leaderboard: | |
| return "No rated outputs yet." | |
| leaderboard_text = "## Top Rated Outputs\n\n" | |
| for i, (output_id, rating) in enumerate(leaderboard): | |
| leaderboard_text += f"{i+1}. Output ID: {output_id[:8]}... - Rating: {rating:.1f}\n" | |
| return leaderboard_text | |
| # Create Gradio interface | |
| with gr.Blocks(theme=gr.themes.Soft( | |
| primary_hue="indigo", | |
| secondary_hue="blue", | |
| neutral_hue="slate", | |
| radius_size="lg", | |
| font=["Inter", "ui-sans-serif", "system-ui", "sans-serif"] | |
| ), | |
| css=""" | |
| .scrollable-notes { | |
| max-height: 300px; | |
| overflow-y: auto; | |
| border: 1px solid #ddd; | |
| padding: 10px; | |
| border-radius: 5px; | |
| } | |
| .good-button { | |
| background-color: #4CAF50 !important; | |
| color: white !important; | |
| } | |
| .fine-button { | |
| background-color: #FFC107 !important; | |
| color: black !important; | |
| } | |
| .bad-button { | |
| background-color: #F44336 !important; | |
| color: white !important; | |
| } | |
| .selected-button { | |
| opacity: 0.8; | |
| box-shadow: 0 0 5px rgba(0,0,0,0.5); | |
| } | |
| .log-message { | |
| padding: 8px; | |
| margin-top: 10px; | |
| border-radius: 4px; | |
| background-color: #f0f0f0; | |
| font-size: 14px; | |
| } | |
| """) as demo: | |
| # Initialize state | |
| state = gr.State(initialize_state()) | |
| # Error message area | |
| error_message = gr.Markdown(visible=False) | |
| log_message = gr.Markdown(visible=False, elem_classes=["log-message"]) | |
| with gr.Tabs(): | |
| with gr.TabItem("Data Labeling"): | |
| # Stage 1: Individual Assessment | |
| with gr.Group(visible=True) as stage1_group: | |
| gr.Markdown("# Output Evaluation") | |
| gr.Markdown("### Please rate the quality of this output") | |
| # Component ID display | |
| component_id_display = gr.Markdown("### Component ID: Loading...") | |
| with gr.Row(): | |
| # Left column for notes - make it scrollable and smaller | |
| with gr.Column(scale=1): | |
| gr.Markdown("### Low Inference Notes") | |
| notes_display = gr.Markdown(elem_classes=["scrollable-notes"]) | |
| # Right column for output and controls | |
| with gr.Column(scale=2): | |
| output_display = gr.Textbox( | |
| label="AI Generated Output", | |
| lines=8, | |
| max_lines=15, | |
| interactive=False | |
| ) | |
| # Rating buttons in a row with colors | |
| with gr.Row(): | |
| good_btn = gr.Button( | |
| "GOOD - Output is excellent", | |
| variant="secondary", | |
| elem_classes=["good-button"] | |
| ) | |
| fine_btn = gr.Button( | |
| "FINE - Output needs minor improvements", | |
| variant="secondary", | |
| elem_classes=["fine-button"] | |
| ) | |
| bad_btn = gr.Button( | |
| "BAD - Output needs major improvements", | |
| variant="secondary", | |
| elem_classes=["bad-button"] | |
| ) | |
| # Explanation input | |
| explanation_input = gr.Textbox( | |
| label="Please explain your choice in one sentence", | |
| lines=2, | |
| placeholder="Explain why you selected this rating..." | |
| ) | |
| # Submit button | |
| submit_btn = gr.Button("Submit and Continue", variant="primary", size="lg") | |
| # Stage 2: Comparative Assessment | |
| with gr.Group(visible=False) as stage2_group: | |
| gr.Markdown("# Compare Outputs") | |
| gr.Markdown("### Which of these two outputs is better?") | |
| # Make notes scrollable here too | |
| gr.Markdown("### Low Inference Notes (Same for both outputs)") | |
| notes_display_elo = gr.Markdown(elem_classes=["scrollable-notes"]) | |
| with gr.Row(): | |
| with gr.Column(): | |
| output_a = gr.Textbox( | |
| label="Output A", | |
| lines=8, | |
| interactive=False | |
| ) | |
| choose_a_btn = gr.Button("Output A is better", variant="secondary") | |
| with gr.Column(): | |
| output_b = gr.Textbox( | |
| label="Output B", | |
| lines=8, | |
| interactive=False | |
| ) | |
| choose_b_btn = gr.Button("Output B is better", variant="secondary") | |
| comparison_explanation = gr.Textbox( | |
| label="Please explain why you chose this output", | |
| lines=2, | |
| placeholder="Explain your preference in one sentence..." | |
| ) | |
| # Submit button for comparison | |
| compare_submit_btn = gr.Button("Submit and Continue", variant="primary", size="lg") | |
| with gr.TabItem("Leaderboard"): | |
| leaderboard_display = gr.Markdown() | |
| refresh_btn = gr.Button("Refresh Leaderboard") | |
| # Load initial data | |
| demo.load( | |
| fn=load_example, | |
| inputs=[state], | |
| outputs=[component_id_display, notes_display, output_display] | |
| ) | |
| # Also load the leaderboard on startup | |
| demo.load( | |
| fn=get_leaderboard_display, | |
| inputs=[], | |
| outputs=[leaderboard_display] | |
| ) | |
| # Connect the buttons to their functions | |
| good_btn.click( | |
| fn=lambda state: select_choice("good", state), | |
| inputs=[state], | |
| outputs=[good_btn, fine_btn, bad_btn, log_message, state] | |
| ) | |
| fine_btn.click( | |
| fn=lambda state: select_choice("fine", state), | |
| inputs=[state], | |
| outputs=[good_btn, fine_btn, bad_btn, log_message, state] | |
| ) | |
| bad_btn.click( | |
| fn=lambda state: select_choice("bad", state), | |
| inputs=[state], | |
| outputs=[good_btn, fine_btn, bad_btn, log_message, state] | |
| ) | |
| submit_btn.click( | |
| fn=stage1_submit, | |
| inputs=[explanation_input, state], | |
| outputs=[ | |
| stage1_group, stage2_group, component_id_display, notes_display, output_display, | |
| explanation_input, error_message, log_message, good_btn, fine_btn, bad_btn, state | |
| ] | |
| ) | |
| choose_a_btn.click( | |
| fn=lambda state: select_comparison_choice("a", state), | |
| inputs=[state], | |
| outputs=[choose_a_btn, choose_b_btn, log_message, state] | |
| ) | |
| choose_b_btn.click( | |
| fn=lambda state: select_comparison_choice("b", state), | |
| inputs=[state], | |
| outputs=[choose_a_btn, choose_b_btn, log_message, state] | |
| ) | |
| compare_submit_btn.click( | |
| fn=stage2_submit, | |
| inputs=[comparison_explanation, state], | |
| outputs=[ | |
| stage1_group, stage2_group, component_id_display, notes_display, output_display, | |
| notes_display_elo, output_a, output_b, comparison_explanation, | |
| error_message, log_message, choose_a_btn, choose_b_btn, state | |
| ] | |
| ) | |
| refresh_btn.click( | |
| fn=get_leaderboard_display, | |
| inputs=[], | |
| outputs=[leaderboard_display] | |
| ) | |
| # For Hugging Face Spaces deployment | |
| if __name__ == "__main__": | |
| # Initialize Argilla on startup | |
| data_utils.init_argilla() | |
| # Launch the app | |
| demo.launch() | |