Spaces:
Runtime error
Runtime error
| # EvalifAI © tutorial | |
| import gradio as gr | |
| from datetime import datetime | |
| from tempfile import TemporaryDirectory | |
| import os | |
| import json | |
| from datasets import load_dataset | |
| import dill | |
| from huggingface_hub import hf_hub_download | |
| from utils import list_rglob_files, load_json_file, create_directory | |
| model_path = hf_hub_download(repo_id='flavienbwk/example-dummy-evaluation', filename='example-dummy-evaluation.dill') | |
| with open(model_path, 'rb') as dill_file: | |
| model = dill.load(dill_file) | |
| def get_occurences(sentence: str) -> dict: | |
| return model(sentence) | |
| def evaluate() -> str: | |
| """ | |
| Dummy evaluation example for EvalifAI © tutorial | |
| This program sends sentences to a model that must | |
| return the number of occurences of the letter "a". | |
| """ | |
| with TemporaryDirectory() as tmpdirname: | |
| absolute_path = os.path.abspath(tmpdirname) | |
| create_directory(absolute_path) | |
| # Step 1 : initializing benchmark results | |
| report = { | |
| "started_at": None, | |
| "ended_at": None, | |
| "score": 0.0, | |
| "success": 0, | |
| "failure": 0, | |
| "metrics": {"valid_responses": 0.0, "success_rate": 0.0}, | |
| } | |
| # Step 2 : checking input data format and gathering sentences | |
| dataset = load_dataset("flavienbwk/example-dummy-evaluation", split="train") | |
| sentences = [sentence for sentence in dataset] | |
| sentences_ht = {} # Storing sentences as a hashtable | |
| for sentence in sentences: | |
| try: | |
| a_occurences = sentence["a_occurences"] | |
| sentences_ht[sentence["sentence"]] = a_occurences | |
| except Exception as e: | |
| report["failure"] += 1 | |
| # Step 3 : benchmarking | |
| report["started_at"] = datetime.now().timestamp() | |
| for sentence, a_occurences in sentences_ht.items(): | |
| try: | |
| a_occurences_response = get_occurences(sentence) | |
| if a_occurences == a_occurences_response: | |
| report["metrics"]["valid_responses"] += 1.0 | |
| report["success"] += 1 | |
| except Exception as e: | |
| report["failure"] += 1 | |
| print(e) | |
| report["metrics"]["success_rate"] = ( | |
| report["metrics"]["valid_responses"] / len(sentences) if sentences else 0.0 | |
| ) | |
| report["score"] = report["metrics"]["success_rate"] | |
| report["ended_at"] = datetime.now().timestamp() | |
| return json.dumps(report, indent=4) | |
| def main(): | |
| """ | |
| Main function to create the Gradio interface. | |
| """ | |
| interface = gr.Interface( | |
| fn=evaluate, | |
| inputs=None, | |
| outputs="text", # Output type is text | |
| title="Example Dummy Evaluation", # Title of the app | |
| description="Click the 'Generate' button to display 'Hello'", # Description | |
| article="This is a simple Gradio app example." | |
| ) | |
| interface.launch() | |
| if __name__ == "__main__": | |
| main() | |