Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -1,136 +1,122 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import json
|
| 3 |
import pandas as pd
|
| 4 |
import numpy as np
|
| 5 |
import torch
|
| 6 |
-
import torch.nn as nn
|
| 7 |
-
import gspread
|
| 8 |
-
import gradio as gr
|
| 9 |
from sklearn.preprocessing import LabelEncoder
|
| 10 |
-
from transformers import AutoTokenizer, AutoModelForSequenceClassification
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
|
| 17 |
-
# 1.
|
| 18 |
-
model_name = 'roberta-large'
|
| 19 |
-
save_path = './trained_roberta-large'
|
| 20 |
-
SHEET_URL = 'https://docs.google.com/spreadsheets/d/12c1ifGn-U6xhNpJ8v7Q_2uxufQHo6uItzmin9lhNqrw/edit?gid=0#gid=0'
|
| 21 |
THEMES = {
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
}
|
| 30 |
|
| 31 |
-
# 2.
|
| 32 |
-
def get_gspread_client():
|
| 33 |
-
service_account_info = os.environ.get('GCP_SERVICE_ACCOUNT')
|
| 34 |
-
if not service_account_info:
|
| 35 |
-
return None
|
| 36 |
-
try:
|
| 37 |
-
info = json.loads(service_account_info)
|
| 38 |
-
scopes = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive']
|
| 39 |
-
creds = Credentials.from_service_account_info(info, scopes=scopes)
|
| 40 |
-
return gspread.authorize(creds)
|
| 41 |
-
except:
|
| 42 |
-
return None
|
| 43 |
-
|
| 44 |
-
try:
|
| 45 |
-
gc = get_gspread_client()
|
| 46 |
-
if gc:
|
| 47 |
-
sh = gc.open_by_url(SHEET_URL)
|
| 48 |
-
else:
|
| 49 |
-
print('Warning: GCP_SERVICE_ACCOUNT not configured.')
|
| 50 |
-
except Exception as e:
|
| 51 |
-
print(f'Gspread connection failed: {e}')
|
| 52 |
-
|
| 53 |
label_encoder = LabelEncoder()
|
| 54 |
label_encoder.fit(list(THEMES.keys()))
|
| 55 |
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
|
|
|
| 60 |
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
|
| 71 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
|
| 73 |
-
#
|
| 74 |
def classify_caption_trained_model(caption):
|
| 75 |
-
inputs = tokenizer(caption, return_tensors=
|
|
|
|
| 76 |
with torch.no_grad():
|
| 77 |
outputs = model(**inputs)
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
|
| 82 |
-
|
|
|
|
| 83 |
df = pd.read_csv(file_obj.name)
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
return f'Saved {theme} to Sheets.'
|
| 94 |
-
except Exception as e: return f'Sheet Error: {str(e)}'
|
| 95 |
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
if sh is None: return 'Error: Cannot retrain without Sheet connection.'
|
| 107 |
-
try:
|
| 108 |
-
data = pd.DataFrame(sh.get_worksheet(0).get_all_records())
|
| 109 |
-
if data.empty: return 'No data.'
|
| 110 |
-
data['labels'] = label_encoder.transform(data['theme'])
|
| 111 |
-
ds = Dataset.from_pandas(data).map(lambda b: tokenizer(b['message'], truncation=True, padding='max_length', max_length=256), batched=True)
|
| 112 |
-
WeightedTrainer(model=model, args=training_args, train_dataset=ds, data_collator=data_collator).train()
|
| 113 |
-
model.save_pretrained(save_path)
|
| 114 |
-
tokenizer.save_pretrained(save_path)
|
| 115 |
-
return 'Retraining complete and model saved locally.'
|
| 116 |
-
except Exception as e: return f'Failed: {e}'
|
| 117 |
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
gr.Button('Classify').click(classify_caption_trained_model, inputs=cap_in, outputs=[res_out, conf_out])
|
| 125 |
-
drop_corr = gr.Dropdown(choices=list(THEMES.keys()), label='Correct Theme')
|
| 126 |
-
status_msg = gr.Markdown()
|
| 127 |
-
gr.Button('Submit Correction').click(save_feedback_to_sheets, inputs=[cap_in, drop_corr], outputs=status_msg)
|
| 128 |
-
with gr.Tab('Bulk Processing'):
|
| 129 |
-
file_in = gr.File()
|
| 130 |
-
out_table = gr.Dataframe(interactive=True)
|
| 131 |
-
gr.Button('Process CSV').click(process_csv_cloud, inputs=file_in, outputs=out_table)
|
| 132 |
-
gr.Button('Sync to Sheets').click(submit_csv_to_sheets, inputs=out_table, outputs=status_msg)
|
| 133 |
-
with gr.Tab('Admin'):
|
| 134 |
-
gr.Button('Sync & Retrain Model', variant='stop').click(sync_and_retrain, outputs=status_msg)
|
| 135 |
|
| 136 |
-
|
|
|
|
|
|
|
|
|
|
| 1 |
import pandas as pd
|
| 2 |
import numpy as np
|
| 3 |
import torch
|
|
|
|
|
|
|
|
|
|
| 4 |
from sklearn.preprocessing import LabelEncoder
|
| 5 |
+
from transformers import AutoTokenizer, AutoModelForSequenceClassification
|
| 6 |
+
import gradio as gr
|
| 7 |
+
import gspread
|
| 8 |
+
from oauth2client.service_account import ServiceAccountCredentials
|
| 9 |
+
import os
|
| 10 |
+
import json
|
| 11 |
|
| 12 |
+
# 1. Define the THEMES dictionary
|
|
|
|
|
|
|
|
|
|
| 13 |
THEMES = {
|
| 14 |
+
"Lifestyle": "Lifestyle: everyday life, hobbies, routines, hanging out, personal moments",
|
| 15 |
+
"Travel & Experience": "Travel & Experience: trips, visiting places, tourism, events, sports games, concerts",
|
| 16 |
+
"Reflection": "Reflection: deep thoughts, life lessons, gratitude, personal growth, emotional reflection",
|
| 17 |
+
"Brand & Business": "Brand & Business: marketing, promoting a brand, entrepreneurship, product announcements, advertisements",
|
| 18 |
+
"Education & Value": "Education & Value: advice, tips, teaching something, sharing knowledge",
|
| 19 |
+
"Relationships": "Relationships: family, friends, romantic, social connections",
|
| 20 |
+
"Performance & Sport": "Performance & Sport: competitions, sports achievements, athletic performance"
|
| 21 |
}
|
| 22 |
|
| 23 |
+
# 2. Initialize LabelEncoder and fit it with THEMES keys
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
label_encoder = LabelEncoder()
|
| 25 |
label_encoder.fit(list(THEMES.keys()))
|
| 26 |
|
| 27 |
+
# 3. Strict Google Sheets Authentication (Environment Variable Only)
|
| 28 |
+
scope = [
|
| 29 |
+
'https://spreadsheets.google.com/feeds',
|
| 30 |
+
'https://www.googleapis.com/auth/drive'
|
| 31 |
+
]
|
| 32 |
|
| 33 |
+
sheet = None
|
| 34 |
+
try:
|
| 35 |
+
creds_json_str = os.environ.get('GDRIVE_SERVICE_ACCOUNT_JSON')
|
| 36 |
+
|
| 37 |
+
if not creds_json_str:
|
| 38 |
+
raise ValueError("CRITICAL ERROR: 'GDRIVE_SERVICE_ACCOUNT_JSON' environment variable not found. Strict secret authentication is required.")
|
| 39 |
+
|
| 40 |
+
# Authenticate using environment variable
|
| 41 |
+
creds_info = json.loads(creds_json_str)
|
| 42 |
+
creds = ServiceAccountCredentials.from_json_keyfile_dict(creds_info, scope)
|
| 43 |
+
client = gspread.authorize(creds)
|
| 44 |
+
sheet = client.open("ClassifierTraining").get_worksheet(0)
|
| 45 |
+
print(f"Successfully authenticated via Environment Variable and connected to sheet: {sheet.title}")
|
| 46 |
+
except Exception as e:
|
| 47 |
+
print(f"Authentication Failure: {e}")
|
| 48 |
|
| 49 |
+
# 4. Load Model and Tokenizer from Hugging Face Hub
|
| 50 |
+
model_name = \"nirvairkataria/caption-classifier-model\"
|
| 51 |
+
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
| 52 |
+
model = AutoModelForSequenceClassification.from_pretrained(model_name)
|
| 53 |
+
model.to(\"cuda\" if torch.cuda.is_available() else \"cpu\")
|
| 54 |
+
model.eval()
|
| 55 |
|
| 56 |
+
# 5. Define classification function
|
| 57 |
def classify_caption_trained_model(caption):
|
| 58 |
+
inputs = tokenizer(caption, return_tensors=\"pt\", truncation=True, padding=True)
|
| 59 |
+
inputs = {k: v.to(model.device) for k, v in inputs.items()}
|
| 60 |
with torch.no_grad():
|
| 61 |
outputs = model(**inputs)
|
| 62 |
+
logits = outputs.logits
|
| 63 |
+
probs = torch.softmax(logits, dim=1).cpu().numpy()[0]
|
| 64 |
+
label_idx = np.argmax(probs)
|
| 65 |
+
pred_label = label_encoder.inverse_transform([label_idx])[0]
|
| 66 |
+
confidence = round(float(probs[label_idx]), 2)
|
| 67 |
+
all_themes = list(THEMES.keys())
|
| 68 |
+
return pred_label, confidence, all_themes
|
| 69 |
+
|
| 70 |
+
# 6. Define feedback submission function
|
| 71 |
+
def save_feedback_to_sheet(caption, corrected_theme):
|
| 72 |
+
if sheet is None:
|
| 73 |
+
return "Error: Google Sheet authentication failed. Feedback cannot be submitted."
|
| 74 |
+
try:
|
| 75 |
+
new_row = [caption, corrected_theme]
|
| 76 |
+
sheet.append_row(new_row)
|
| 77 |
+
return "Feedback submitted successfully!"
|
| 78 |
+
except Exception as e:
|
| 79 |
+
return f"Failed to submit feedback: {e}"
|
| 80 |
|
| 81 |
+
# 7. Define CSV processing function
|
| 82 |
+
def process_csv(file_obj):
|
| 83 |
df = pd.read_csv(file_obj.name)
|
| 84 |
+
predicted_themes, confidences = [], []
|
| 85 |
+
for caption in df['message']:
|
| 86 |
+
theme, conf, _ = classify_caption_trained_model(caption)
|
| 87 |
+
predicted_themes.append(theme)
|
| 88 |
+
confidences.append(conf)
|
| 89 |
+
df['Predicted Theme'] = predicted_themes
|
| 90 |
+
df['Confidence'] = confidences
|
| 91 |
+
output_file = \"classified_results.csv\"
|
| 92 |
+
df.to_csv(output_file, index=False)
|
| 93 |
+
return df, output_file
|
| 94 |
|
| 95 |
+
# 8. Gradio UI logic
|
| 96 |
+
def handle_classification(caption):
|
| 97 |
+
pred_label, confidence, all_themes = classify_caption_trained_model(caption)
|
| 98 |
+
return pred_label, confidence, gr.update(choices=all_themes, value=pred_label)
|
|
|
|
|
|
|
| 99 |
|
| 100 |
+
with gr.Blocks() as demo:
|
| 101 |
+
gr.Markdown(\"# Caption Classifier (Strict Auth Mode)\")
|
| 102 |
+
with gr.Tab(\"Single Caption\"):
|
| 103 |
+
caption_input = gr.Textbox(label=\"Enter Caption\", lines=4)
|
| 104 |
+
with gr.Row():
|
| 105 |
+
theme_output = gr.Textbox(label=\"Predicted Theme\", interactive=False)
|
| 106 |
+
confidence_output = gr.Textbox(label=\"Confidence\", interactive=False)
|
| 107 |
+
classify_btn = gr.Button(\"Classify\")
|
| 108 |
+
corrected_theme_dropdown = gr.Dropdown(label=\"Corrected Theme\", choices=list(THEMES.keys()))
|
| 109 |
+
submit_btn = gr.Button(\"Submit Correction\")
|
| 110 |
+
status_output = gr.Textbox(label=\"Status\", interactive=False)
|
| 111 |
|
| 112 |
+
classify_btn.click(handle_classification, inputs=caption_input, outputs=[theme_output, confidence_output, corrected_theme_dropdown])
|
| 113 |
+
submit_btn.click(save_feedback_to_sheet, inputs=[caption_input, corrected_theme_dropdown], outputs=status_output)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 114 |
|
| 115 |
+
with gr.Tab(\"CSV Upload\"):
|
| 116 |
+
file_input = gr.File(label=\"Upload CSV\")
|
| 117 |
+
output_table = gr.Dataframe()
|
| 118 |
+
download_file = gr.File()
|
| 119 |
+
process_btn = gr.Button(\"Run\")
|
| 120 |
+
process_btn.click(process_csv, inputs=file_input, outputs=[output_table, download_file])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
|
| 122 |
+
demo.launch()
|