import io import os from fastapi import FastAPI, HTTPException, Request from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates from fastapi.middleware.gzip import GZipMiddleware from fastapi.middleware.cors import CORSMiddleware from pydantic import BaseModel import pickle import torch import torch.nn.functional as F import csv from transformers import AutoModelForSequenceClassification, AutoTokenizer from datasets import load_dataset import emoji from wordcloud import WordCloud from fastapi.responses import StreamingResponse import matplotlib.pyplot as plt from sklearn.feature_extraction.text import CountVectorizer from sklearn.model_selection import train_test_split from sklearn.naive_bayes import MultinomialNB from sklearn.svm import SVC from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay app = FastAPI() app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) # Global cache hf_dataset_cache = None @app.on_event("startup") async def load_hf_dataset_once(): global hf_dataset_cache try: hf_dataset_cache = load_dataset("KagChi/indonesian-gambling-words", split="test") print("Hugging Face dataset loaded and cached.") except Exception as e: print(f"Failed to load HF dataset on startup: {e}") hf_dataset_cache = None app.mount("/static",StaticFiles(directory="static"),name="static") app.add_middleware(GZipMiddleware) templates = Jinja2Templates(directory="templates") original_torch_load = torch.load def torch_load_cpu(*args, **kwargs): if 'map_location' not in kwargs: kwargs['map_location'] = torch.device('cpu') return original_torch_load(*args, **kwargs) torch.load = torch_load_cpu pkl_model_path = "models/model.pkl" pkl_tokenizer_path = "models/tokenizer.pkl" use_pickle = os.path.exists(pkl_model_path) and os.path.exists(pkl_tokenizer_path) if use_pickle: print("Loading model from pickle files...") with open(pkl_model_path, "rb") as f: model = pickle.load(f) with open(pkl_tokenizer_path, "rb") as f: tokenizer = pickle.load(f) else: print("Loading model from Hugging Face...") model_name = "KagChi/indonesian-sentiment-gambling-classification" tokenizer = AutoTokenizer.from_pretrained(model_name) model = AutoModelForSequenceClassification.from_pretrained(model_name) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.to(device) model.eval() # Pydantic models for request/response class PredictionInput(BaseModel): text: str # Text input for the model class PredictionResponse(BaseModel): prediction: int # Predicted class (-1: unknown, 0: not gambling, 1: gambling) prediction_text: str # Human readable prediction probabilities: dict # Probabilities for each class confidence: float # Probability for the predicted class status: str # "judi", "bukan judi", or "ragu-ragu" highlighted_tokens: list[str] def get_important_tokens(text: str, tokenizer, model, target_class=1, top_k=3): model.eval() # Tokenize dengan offset mapping untuk kata asli encoding = tokenizer( text, return_tensors="pt", return_offsets_mapping=True, padding=True, truncation=True, max_length=128 ) input_ids = encoding["input_ids"].to(model.device) attention_mask = encoding["attention_mask"].to(model.device) offset_mapping = encoding["offset_mapping"][0].tolist() # Dapatkan embeddings embedding_layer = model.get_input_embeddings() input_embeddings = embedding_layer(input_ids) input_embeddings.retain_grad() # Forward pass outputs = model(inputs_embeds=input_embeddings, attention_mask=attention_mask) probs = F.softmax(outputs.logits, dim=-1) target_score = probs[0][target_class] # Backward pass model.zero_grad() target_score.backward() # Importance score per token grads = input_embeddings.grad.abs().sum(dim=-1).squeeze().tolist() # Mapping: offset range → importance score token_scores = [] for i, (start, end) in enumerate(offset_mapping): if start == end: # Special tokens continue token_scores.append((start, end, grads[i])) # Gabungkan berdasarkan overlap character range from collections import defaultdict char_score_map = defaultdict(float) for start, end, score in token_scores: for i in range(start, end): char_score_map[i] += score # Bagi teks ke kata import re word_spans = [(m.start(), m.end(), m.group()) for m in re.finditer(r'\S+', text)] word_scores = [] for start, end, word in word_spans: score = sum(char_score_map[i] for i in range(start, end)) word_scores.append((word, score)) # Ambil kata dengan skor tertinggi word_scores.sort(key=lambda x: x[1], reverse=True) highlighted_words = [w for w, s in word_scores[:top_k]] return highlighted_words @app.get("/") def read_root(request: Request): return templates.TemplateResponse( request=request, name="index.html", context={"id": id} ) @app.get("/visualize") async def visualize(request: Request): return templates.TemplateResponse("visualize.html", {"request": request}) @app.get("/wordcloud") async def generate_wordcloud(): global hf_dataset_cache if hf_dataset_cache: # Filter: only samples with text AND label == 1 (Gambling) filtered = [ sample["text"] for sample in hf_dataset_cache if sample.get("text") and sample.get("label") == 1 and sample["text"].strip() ] text_corpus = " ".join(filtered) else: text_corpus = "" # fallback if empty if not text_corpus.strip(): text_corpus = "judi bola togel slot kasino poker" wordcloud = WordCloud(width=800, height=400, background_color='white').generate(text_corpus) buf = io.BytesIO() plt.figure(figsize=(10, 5)) plt.imshow(wordcloud, interpolation='bilinear') plt.axis("off") plt.tight_layout() plt.savefig(buf, format='png') buf.seek(0) return StreamingResponse(buf, media_type="image/png") @app.get("/confusion-matrix/{model_type}") async def confusion_matrix_image(model_type: str): global hf_dataset_cache if hf_dataset_cache is None: raise HTTPException(status_code=500, detail="Dataset not loaded") # Include all labeled samples (both 0 and 1) filtered_samples = [ sample for sample in hf_dataset_cache if sample.get("text") and sample["text"].strip() ] if not filtered_samples: raise HTTPException(status_code=400, detail="No valid samples found.") texts = [sample["text"] for sample in filtered_samples] labels = [sample["label"] for sample in filtered_samples] vectorizer = CountVectorizer() X = vectorizer.fit_transform(texts) y = labels X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) if model_type == "naive-bayes": clf = MultinomialNB() elif model_type == "svm": clf = SVC() else: raise HTTPException(status_code=400, detail="Invalid model type (use 'naive-bayes' or 'svm')") clf.fit(X_train, y_train) y_pred = clf.predict(X_test) cm = confusion_matrix(y_test, y_pred, labels=[0, 1]) disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=["Bukan Judi", "Judi"]) buf = io.BytesIO() disp.plot(cmap='Blues') plt.tight_layout() plt.savefig(buf, format="png") plt.close() buf.seek(0) return StreamingResponse(buf, media_type="image/png") @app.post("/predict", response_model=PredictionResponse) async def predict(input_data: PredictionInput): try: threshold_low = 0.4 threshold_high = 0.7 text = str(input_data.text) if input_data.text else "" text = emoji.replace_emoji(text, replace='').strip() # Tokenize input text inputs = tokenizer( text, return_tensors="pt", padding=True, truncation=True, max_length=128 ) inputs = {key: val.to(device) for key, val in inputs.items()} # Make prediction with torch.no_grad(): outputs = model(**inputs) probs = F.softmax(outputs.logits, dim=-1) prob_0 = probs[0][0].item() prob_1 = probs[0][1].item() # Interpret prediction if prob_1 >= threshold_high: prediction = 1 prediction_text = f'judi ({prob_1:.2f})' status = "judi" confidence = prob_1 elif prob_1 <= threshold_low: prediction = 0 prediction_text = f'bukan judi ({prob_0:.2f})' status = "bukan judi" confidence = prob_0 else: prediction = -1 prediction_text = f'ragu-ragu ({prob_1:.2f})' status = "ragu-ragu" confidence = max(prob_0, prob_1) highlighted_tokens = get_important_tokens(text, tokenizer, model, target_class=1) return PredictionResponse( prediction=prediction, prediction_text=prediction_text, probabilities={"prob_bukan_judi": prob_0, "prob_judi": prob_1}, confidence=confidence, status=status, highlighted_tokens=highlighted_tokens ) except Exception as e: raise HTTPException(status_code=400, detail=f"Prediction error: {str(e)}") @app.get("/predict") async def predict_get(request: Request): """GET endpoint to render prediction form""" return templates.TemplateResponse( request=request, name="predict.html", context={} ) @app.get("/samples") async def get_samples(): file_path = "static/samples.csv" text_field = "text" # Try loading from local CSV first try: if os.path.exists(file_path): with open(file_path, newline='', encoding='utf-8') as csvfile: reader = csv.DictReader(csvfile) samples = [row["text"] for row in reader if row.get("text")] return samples except Exception as e: # Log CSV read failure and fallback print(f"Failed to read CSV: {e}") # Fallback to Hugging Face dataset global hf_dataset_cache if hf_dataset_cache is not None: samples = [example[text_field] for example in hf_dataset_cache if text_field in example] return samples else: raise HTTPException(status_code=500, detail="Hugging Face dataset is not available.")