Spaces:
Runtime error
Runtime error
Download main.py from KagChi/ISGDModelDemo: direct link, hf CLI and curl.
- Browser
- Download file 10.7 kB
-
https://huggingface.co/spaces/KagChi/ISGDModelDemo/resolve/main/main.py
- Command line
-
hf download hf://spaces/KagChi/ISGDModelDemo/main.py
-
curl -L -o main.py https://huggingface.co/spaces/KagChi/ISGDModelDemo/resolve/main/main.py
10.7 kB
| import io | |
| import os | |
| from fastapi import FastAPI, HTTPException, Request | |
| from fastapi.staticfiles import StaticFiles | |
| from fastapi.templating import Jinja2Templates | |
| from fastapi.middleware.gzip import GZipMiddleware | |
| from fastapi.middleware.cors import CORSMiddleware | |
| from pydantic import BaseModel | |
| import pickle | |
| import torch | |
| import torch.nn.functional as F | |
| import csv | |
| from transformers import AutoModelForSequenceClassification, AutoTokenizer | |
| from datasets import load_dataset | |
| import emoji | |
| from wordcloud import WordCloud | |
| from fastapi.responses import StreamingResponse | |
| import matplotlib.pyplot as plt | |
| from sklearn.feature_extraction.text import CountVectorizer | |
| from sklearn.model_selection import train_test_split | |
| from sklearn.naive_bayes import MultinomialNB | |
| from sklearn.svm import SVC | |
| from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay | |
| app = FastAPI() | |
| app.add_middleware( | |
| CORSMiddleware, | |
| allow_origins=["*"], | |
| allow_credentials=True, | |
| allow_methods=["*"], | |
| allow_headers=["*"], | |
| ) | |
| # Global cache | |
| hf_dataset_cache = None | |
| async def load_hf_dataset_once(): | |
| global hf_dataset_cache | |
| try: | |
| hf_dataset_cache = load_dataset("KagChi/indonesian-gambling-words", split="test") | |
| print("Hugging Face dataset loaded and cached.") | |
| except Exception as e: | |
| print(f"Failed to load HF dataset on startup: {e}") | |
| hf_dataset_cache = None | |
| app.mount("/static",StaticFiles(directory="static"),name="static") | |
| app.add_middleware(GZipMiddleware) | |
| templates = Jinja2Templates(directory="templates") | |
| original_torch_load = torch.load | |
| def torch_load_cpu(*args, **kwargs): | |
| if 'map_location' not in kwargs: | |
| kwargs['map_location'] = torch.device('cpu') | |
| return original_torch_load(*args, **kwargs) | |
| torch.load = torch_load_cpu | |
| pkl_model_path = "models/model.pkl" | |
| pkl_tokenizer_path = "models/tokenizer.pkl" | |
| use_pickle = os.path.exists(pkl_model_path) and os.path.exists(pkl_tokenizer_path) | |
| if use_pickle: | |
| print("Loading model from pickle files...") | |
| with open(pkl_model_path, "rb") as f: | |
| model = pickle.load(f) | |
| with open(pkl_tokenizer_path, "rb") as f: | |
| tokenizer = pickle.load(f) | |
| else: | |
| print("Loading model from Hugging Face...") | |
| model_name = "KagChi/indonesian-sentiment-gambling-classification" | |
| tokenizer = AutoTokenizer.from_pretrained(model_name) | |
| model = AutoModelForSequenceClassification.from_pretrained(model_name) | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| model.to(device) | |
| model.eval() | |
| # Pydantic models for request/response | |
| class PredictionInput(BaseModel): | |
| text: str # Text input for the model | |
| class PredictionResponse(BaseModel): | |
| prediction: int # Predicted class (-1: unknown, 0: not gambling, 1: gambling) | |
| prediction_text: str # Human readable prediction | |
| probabilities: dict # Probabilities for each class | |
| confidence: float # Probability for the predicted class | |
| status: str # "judi", "bukan judi", or "ragu-ragu" | |
| highlighted_tokens: list[str] | |
| def get_important_tokens(text: str, tokenizer, model, target_class=1, top_k=3): | |
| model.eval() | |
| # Tokenize dengan offset mapping untuk kata asli | |
| encoding = tokenizer( | |
| text, | |
| return_tensors="pt", | |
| return_offsets_mapping=True, | |
| padding=True, | |
| truncation=True, | |
| max_length=128 | |
| ) | |
| input_ids = encoding["input_ids"].to(model.device) | |
| attention_mask = encoding["attention_mask"].to(model.device) | |
| offset_mapping = encoding["offset_mapping"][0].tolist() | |
| # Dapatkan embeddings | |
| embedding_layer = model.get_input_embeddings() | |
| input_embeddings = embedding_layer(input_ids) | |
| input_embeddings.retain_grad() | |
| # Forward pass | |
| outputs = model(inputs_embeds=input_embeddings, attention_mask=attention_mask) | |
| probs = F.softmax(outputs.logits, dim=-1) | |
| target_score = probs[0][target_class] | |
| # Backward pass | |
| model.zero_grad() | |
| target_score.backward() | |
| # Importance score per token | |
| grads = input_embeddings.grad.abs().sum(dim=-1).squeeze().tolist() | |
| # Mapping: offset range → importance score | |
| token_scores = [] | |
| for i, (start, end) in enumerate(offset_mapping): | |
| if start == end: # Special tokens | |
| continue | |
| token_scores.append((start, end, grads[i])) | |
| # Gabungkan berdasarkan overlap character range | |
| from collections import defaultdict | |
| char_score_map = defaultdict(float) | |
| for start, end, score in token_scores: | |
| for i in range(start, end): | |
| char_score_map[i] += score | |
| # Bagi teks ke kata | |
| import re | |
| word_spans = [(m.start(), m.end(), m.group()) for m in re.finditer(r'\S+', text)] | |
| word_scores = [] | |
| for start, end, word in word_spans: | |
| score = sum(char_score_map[i] for i in range(start, end)) | |
| word_scores.append((word, score)) | |
| # Ambil kata dengan skor tertinggi | |
| word_scores.sort(key=lambda x: x[1], reverse=True) | |
| highlighted_words = [w for w, s in word_scores[:top_k]] | |
| return highlighted_words | |
| def read_root(request: Request): | |
| return templates.TemplateResponse( | |
| request=request, name="index.html", context={"id": id} | |
| ) | |
| async def visualize(request: Request): | |
| return templates.TemplateResponse("visualize.html", {"request": request}) | |
| async def generate_wordcloud(): | |
| global hf_dataset_cache | |
| if hf_dataset_cache: | |
| # Filter: only samples with text AND label == 1 (Gambling) | |
| filtered = [ | |
| sample["text"] | |
| for sample in hf_dataset_cache | |
| if sample.get("text") and sample.get("label") == 1 and sample["text"].strip() | |
| ] | |
| text_corpus = " ".join(filtered) | |
| else: | |
| text_corpus = "" | |
| # fallback if empty | |
| if not text_corpus.strip(): | |
| text_corpus = "judi bola togel slot kasino poker" | |
| wordcloud = WordCloud(width=800, height=400, background_color='white').generate(text_corpus) | |
| buf = io.BytesIO() | |
| plt.figure(figsize=(10, 5)) | |
| plt.imshow(wordcloud, interpolation='bilinear') | |
| plt.axis("off") | |
| plt.tight_layout() | |
| plt.savefig(buf, format='png') | |
| buf.seek(0) | |
| return StreamingResponse(buf, media_type="image/png") | |
| async def confusion_matrix_image(model_type: str): | |
| global hf_dataset_cache | |
| if hf_dataset_cache is None: | |
| raise HTTPException(status_code=500, detail="Dataset not loaded") | |
| # Include all labeled samples (both 0 and 1) | |
| filtered_samples = [ | |
| sample for sample in hf_dataset_cache | |
| if sample.get("text") and sample["text"].strip() | |
| ] | |
| if not filtered_samples: | |
| raise HTTPException(status_code=400, detail="No valid samples found.") | |
| texts = [sample["text"] for sample in filtered_samples] | |
| labels = [sample["label"] for sample in filtered_samples] | |
| vectorizer = CountVectorizer() | |
| X = vectorizer.fit_transform(texts) | |
| y = labels | |
| X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) | |
| if model_type == "naive-bayes": | |
| clf = MultinomialNB() | |
| elif model_type == "svm": | |
| clf = SVC() | |
| else: | |
| raise HTTPException(status_code=400, detail="Invalid model type (use 'naive-bayes' or 'svm')") | |
| clf.fit(X_train, y_train) | |
| y_pred = clf.predict(X_test) | |
| cm = confusion_matrix(y_test, y_pred, labels=[0, 1]) | |
| disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=["Bukan Judi", "Judi"]) | |
| buf = io.BytesIO() | |
| disp.plot(cmap='Blues') | |
| plt.tight_layout() | |
| plt.savefig(buf, format="png") | |
| plt.close() | |
| buf.seek(0) | |
| return StreamingResponse(buf, media_type="image/png") | |
| async def predict(input_data: PredictionInput): | |
| try: | |
| threshold_low = 0.4 | |
| threshold_high = 0.7 | |
| text = str(input_data.text) if input_data.text else "" | |
| text = emoji.replace_emoji(text, replace='').strip() | |
| # Tokenize input text | |
| inputs = tokenizer( | |
| text, | |
| return_tensors="pt", | |
| padding=True, | |
| truncation=True, | |
| max_length=128 | |
| ) | |
| inputs = {key: val.to(device) for key, val in inputs.items()} | |
| # Make prediction | |
| with torch.no_grad(): | |
| outputs = model(**inputs) | |
| probs = F.softmax(outputs.logits, dim=-1) | |
| prob_0 = probs[0][0].item() | |
| prob_1 = probs[0][1].item() | |
| # Interpret prediction | |
| if prob_1 >= threshold_high: | |
| prediction = 1 | |
| prediction_text = f'judi ({prob_1:.2f})' | |
| status = "judi" | |
| confidence = prob_1 | |
| elif prob_1 <= threshold_low: | |
| prediction = 0 | |
| prediction_text = f'bukan judi ({prob_0:.2f})' | |
| status = "bukan judi" | |
| confidence = prob_0 | |
| else: | |
| prediction = -1 | |
| prediction_text = f'ragu-ragu ({prob_1:.2f})' | |
| status = "ragu-ragu" | |
| confidence = max(prob_0, prob_1) | |
| highlighted_tokens = get_important_tokens(text, tokenizer, model, target_class=1) | |
| return PredictionResponse( | |
| prediction=prediction, | |
| prediction_text=prediction_text, | |
| probabilities={"prob_bukan_judi": prob_0, "prob_judi": prob_1}, | |
| confidence=confidence, | |
| status=status, | |
| highlighted_tokens=highlighted_tokens | |
| ) | |
| except Exception as e: | |
| raise HTTPException(status_code=400, detail=f"Prediction error: {str(e)}") | |
| async def predict_get(request: Request): | |
| """GET endpoint to render prediction form""" | |
| return templates.TemplateResponse( | |
| request=request, name="predict.html", context={} | |
| ) | |
| async def get_samples(): | |
| file_path = "static/samples.csv" | |
| text_field = "text" | |
| # Try loading from local CSV first | |
| try: | |
| if os.path.exists(file_path): | |
| with open(file_path, newline='', encoding='utf-8') as csvfile: | |
| reader = csv.DictReader(csvfile) | |
| samples = [row["text"] for row in reader if row.get("text")] | |
| return samples | |
| except Exception as e: | |
| # Log CSV read failure and fallback | |
| print(f"Failed to read CSV: {e}") | |
| # Fallback to Hugging Face dataset | |
| global hf_dataset_cache | |
| if hf_dataset_cache is not None: | |
| samples = [example[text_field] for example in hf_dataset_cache if text_field in example] | |
| return samples | |
| else: | |
| raise HTTPException(status_code=500, detail="Hugging Face dataset is not available.") |