Upload folder using huggingface_hub

app.py

@@ -19,4 +19,4 @@ demo = gr.Interface(
     ]
 )
 
-demo.launch(share= True)
+demo.launch(server_name="0.0.0.0", server_port=7860)

evaluation.py

@@ -0,0 +1,26 @@
+from sklearn.metrics import accuracy_score, f1_score
+import numpy as np
+from datasets import load_dataset
+from model_setup import predict
+
+SAMPLE_SIZE = 100
+
+
+def evaluate_model(sample_size=SAMPLE_SIZE):
+    """Score the model on a slice of the test set, return accuracy and weighted F1."""
+    dataset = load_dataset("cardiffnlp/tweet_eval", "sentiment")
+    test_dataset = dataset["test"]
+
+    texts = test_dataset["text"][:sample_size]
+    true_labels = np.array(test_dataset["label"][:sample_size], dtype=int)
+    pred_labels = np.array([predict(text) for text in texts], dtype=int)
+
+    accuracy = accuracy_score(true_labels, pred_labels)
+    f1 = f1_score(true_labels, pred_labels, average="weighted")
+    return accuracy, f1
+
+
+if __name__ == "__main__":
+    accuracy, f1 = evaluate_model()
+    print(f"{'Accuracy:':<12}{accuracy:>8.4f}")
+    print(f"{'F1 score:':<12}{f1:>8.4f}")

model_setup.py

@@ -2,13 +2,6 @@ from transformers import AutoModelForSequenceClassification
 from transformers import AutoTokenizer, AutoConfig
 import numpy as np
 from scipy.special import softmax
-from datasets import load_dataset
-
-'''
-Load the dataset tweet_eval with sentiment task
-train / validation / test already Splitted
-'''
-dataset = load_dataset("cardiffnlp/tweet_eval", "sentiment")
 
 # Preprocess text as required by the model
 # - @username -> @user
@@ -34,13 +27,5 @@ def predict(text):
     text = preprocess(text)
     encoded_input = tokenizer(text, return_tensors='pt')
     output = model(**encoded_input)
-    scores = output[0][0].detach().numpy()
-    scores = softmax(scores)
-
-    ranking = np.argsort(scores)[::-1]
-    top_idx = int(ranking[0])
-    top_label = config.id2label[top_idx]
-    top_score = np.round(float(scores[top_idx]), 4)
-    print(f"Prediction: {top_label} ({top_score})")
-
-    return top_idx
+    scores = softmax(output[0][0].detach().numpy())
+    return int(np.argsort(scores)[::-1][0])

monitoring.py

@@ -1,6 +1,9 @@
-from model_setup import dataset, predict
 import numpy as np
 import random
+from datasets import load_dataset
+from model_setup import predict
+
+dataset = load_dataset("cardiffnlp/tweet_eval", "sentiment")
 
 # Take a sample of random tweets to monitor
 test_dataset = dataset["test"]