Spaces:
Runtime error
Runtime error
File size: 2,983 Bytes
238b11f 8c4fa0a 238b11f 2ead08f 238b11f 8c4fa0a 2ead08f 238b11f 2ead08f 238b11f 39f3543 238b11f 39f3543 238b11f 8c4fa0a 39f3543 238b11f 39f3543 238b11f 39f3543 238b11f 39f3543 238b11f 39f3543 afe2b65 39f3543 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | # EvalifAI © tutorial
import gradio as gr
from datetime import datetime
from tempfile import TemporaryDirectory
import os
import json
from datasets import load_dataset
import dill
from huggingface_hub import hf_hub_download
from utils import list_rglob_files, load_json_file, create_directory
model_path = hf_hub_download(repo_id='flavienbwk/example-dummy-evaluation', filename='example-dummy-evaluation.dill')
with open(model_path, 'rb') as dill_file:
model = dill.load(dill_file)
def get_occurences(sentence: str) -> dict:
return model(sentence)
def evaluate() -> str:
"""
Dummy evaluation example for EvalifAI © tutorial
This program sends sentences to a model that must
return the number of occurences of the letter "a".
"""
with TemporaryDirectory() as tmpdirname:
absolute_path = os.path.abspath(tmpdirname)
create_directory(absolute_path)
# Step 1 : initializing benchmark results
report = {
"started_at": None,
"ended_at": None,
"score": 0.0,
"success": 0,
"failure": 0,
"metrics": {"valid_responses": 0.0, "success_rate": 0.0},
}
# Step 2 : checking input data format and gathering sentences
dataset = load_dataset("flavienbwk/example-dummy-evaluation", split="train")
sentences = [sentence for sentence in dataset]
sentences_ht = {} # Storing sentences as a hashtable
for sentence in sentences:
try:
a_occurences = sentence["a_occurences"]
sentences_ht[sentence["sentence"]] = a_occurences
except Exception as e:
report["failure"] += 1
# Step 3 : benchmarking
report["started_at"] = datetime.now().timestamp()
for sentence, a_occurences in sentences_ht.items():
try:
a_occurences_response = get_occurences(sentence)
if a_occurences == a_occurences_response:
report["metrics"]["valid_responses"] += 1.0
report["success"] += 1
except Exception as e:
report["failure"] += 1
print(e)
report["metrics"]["success_rate"] = (
report["metrics"]["valid_responses"] / len(sentences) if sentences else 0.0
)
report["score"] = report["metrics"]["success_rate"]
report["ended_at"] = datetime.now().timestamp()
return json.dumps(report, indent=4)
def main():
"""
Main function to create the Gradio interface.
"""
interface = gr.Interface(
fn=evaluate,
inputs=None,
outputs="text", # Output type is text
title="Example Dummy Evaluation", # Title of the app
description="Click the 'Generate' button to display 'Hello'", # Description
article="This is a simple Gradio app example."
)
interface.launch()
if __name__ == "__main__":
main()
|