File size: 10,744 Bytes
0e70368
fe115ca
 
 
 
 
 
 
 
 
 
 
9749881
 
a645675
0e70368
 
 
 
 
 
 
 
fe115ca
 
 
 
 
 
 
 
 
 
 
9749881
 
 
 
 
 
 
 
 
 
 
 
 
fe115ca
 
 
 
 
 
 
 
 
 
 
9749881
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fe115ca
 
 
 
 
 
 
 
 
 
 
 
 
 
42d91ef
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fe115ca
 
 
 
 
 
 
0e70368
 
 
 
 
 
 
 
5697faf
0e70368
5697faf
 
 
 
 
 
 
 
 
0e70368
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ebd3a5d
5697faf
 
ebd3a5d
5697faf
 
 
ebd3a5d
5697faf
 
 
0e70368
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ebd3a5d
5697faf
0e70368
 
 
 
 
5697faf
0e70368
 
 
fe115ca
 
 
 
 
42d91ef
 
a645675
42d91ef
fe115ca
 
42d91ef
 
 
 
fe115ca
 
 
 
 
 
 
 
42d91ef
 
 
 
fe115ca
42d91ef
fe115ca
 
 
 
42d91ef
fe115ca
 
 
 
42d91ef
fe115ca
 
 
42d91ef
 
 
fe115ca
 
 
 
 
42d91ef
 
fe115ca
42d91ef
fe115ca
 
 
 
 
 
 
 
 
 
 
 
 
9749881
fe115ca
9749881
fe115ca
9749881
 
 
 
 
fe115ca
9749881
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
import io
import os
from fastapi import FastAPI, HTTPException, Request
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from fastapi.middleware.gzip import GZipMiddleware
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
import pickle
import torch
import torch.nn.functional as F
import csv
from transformers import AutoModelForSequenceClassification, AutoTokenizer
from datasets import load_dataset
import emoji
from wordcloud import WordCloud
from fastapi.responses import StreamingResponse
import matplotlib.pyplot as plt
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.svm import SVC
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay

app = FastAPI()

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Global cache
hf_dataset_cache = None

@app.on_event("startup")
async def load_hf_dataset_once():
    global hf_dataset_cache
    try:
        hf_dataset_cache = load_dataset("KagChi/indonesian-gambling-words", split="test")
        print("Hugging Face dataset loaded and cached.")
    except Exception as e:
        print(f"Failed to load HF dataset on startup: {e}")
        hf_dataset_cache = None

app.mount("/static",StaticFiles(directory="static"),name="static")
app.add_middleware(GZipMiddleware)
templates = Jinja2Templates(directory="templates")

original_torch_load = torch.load
def torch_load_cpu(*args, **kwargs):
    if 'map_location' not in kwargs:
        kwargs['map_location'] = torch.device('cpu')
    return original_torch_load(*args, **kwargs)
torch.load = torch_load_cpu

pkl_model_path = "models/model.pkl"
pkl_tokenizer_path = "models/tokenizer.pkl"

use_pickle = os.path.exists(pkl_model_path) and os.path.exists(pkl_tokenizer_path)

if use_pickle:
    print("Loading model from pickle files...")
    with open(pkl_model_path, "rb") as f:
        model = pickle.load(f)
    with open(pkl_tokenizer_path, "rb") as f:
        tokenizer = pickle.load(f)
else:
    print("Loading model from Hugging Face...")
    model_name = "KagChi/indonesian-sentiment-gambling-classification"
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForSequenceClassification.from_pretrained(model_name)
    
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)
model.eval()

# Pydantic models for request/response
class PredictionInput(BaseModel):
    text: str  # Text input for the model
    
class PredictionResponse(BaseModel):
    prediction: int  # Predicted class (-1: unknown, 0: not gambling, 1: gambling)
    prediction_text: str  # Human readable prediction
    probabilities: dict  # Probabilities for each class
    confidence: float  # Probability for the predicted class
    status: str  # "judi", "bukan judi", or "ragu-ragu"
    highlighted_tokens: list[str]

def get_important_tokens(text: str, tokenizer, model, target_class=1, top_k=3):
    model.eval()

    # Tokenize dengan offset mapping untuk kata asli
    encoding = tokenizer(
        text,
        return_tensors="pt",
        return_offsets_mapping=True,
        padding=True,
        truncation=True,
        max_length=128
    )
    input_ids = encoding["input_ids"].to(model.device)
    attention_mask = encoding["attention_mask"].to(model.device)
    offset_mapping = encoding["offset_mapping"][0].tolist()

    # Dapatkan embeddings
    embedding_layer = model.get_input_embeddings()
    input_embeddings = embedding_layer(input_ids)
    input_embeddings.retain_grad()

    # Forward pass
    outputs = model(inputs_embeds=input_embeddings, attention_mask=attention_mask)
    probs = F.softmax(outputs.logits, dim=-1)
    target_score = probs[0][target_class]

    # Backward pass
    model.zero_grad()
    target_score.backward()

    # Importance score per token
    grads = input_embeddings.grad.abs().sum(dim=-1).squeeze().tolist()

    # Mapping: offset range → importance score
    token_scores = []
    for i, (start, end) in enumerate(offset_mapping):
        if start == end:  # Special tokens
            continue
        token_scores.append((start, end, grads[i]))

    # Gabungkan berdasarkan overlap character range
    from collections import defaultdict
    char_score_map = defaultdict(float)
    for start, end, score in token_scores:
        for i in range(start, end):
            char_score_map[i] += score

    # Bagi teks ke kata
    import re
    word_spans = [(m.start(), m.end(), m.group()) for m in re.finditer(r'\S+', text)]

    word_scores = []
    for start, end, word in word_spans:
        score = sum(char_score_map[i] for i in range(start, end))
        word_scores.append((word, score))

    # Ambil kata dengan skor tertinggi
    word_scores.sort(key=lambda x: x[1], reverse=True)
    highlighted_words = [w for w, s in word_scores[:top_k]]

    return highlighted_words

@app.get("/")
def read_root(request: Request):
    return templates.TemplateResponse(
        request=request, name="index.html", context={"id": id}
    )

@app.get("/visualize")
async def visualize(request: Request):
    return templates.TemplateResponse("visualize.html", {"request": request})


@app.get("/wordcloud")
async def generate_wordcloud():
    global hf_dataset_cache

    if hf_dataset_cache:
        # Filter: only samples with text AND label == 1 (Gambling)
        filtered = [
            sample["text"]
            for sample in hf_dataset_cache
            if sample.get("text") and sample.get("label") == 1 and sample["text"].strip()
        ]
        text_corpus = " ".join(filtered)
    else:
        text_corpus = ""

    # fallback if empty
    if not text_corpus.strip():
        text_corpus = "judi bola togel slot kasino poker"

    wordcloud = WordCloud(width=800, height=400, background_color='white').generate(text_corpus)

    buf = io.BytesIO()
    plt.figure(figsize=(10, 5))
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis("off")
    plt.tight_layout()
    plt.savefig(buf, format='png')
    buf.seek(0)
    return StreamingResponse(buf, media_type="image/png")

@app.get("/confusion-matrix/{model_type}")
async def confusion_matrix_image(model_type: str):
    global hf_dataset_cache
    if hf_dataset_cache is None:
        raise HTTPException(status_code=500, detail="Dataset not loaded")

    # Include all labeled samples (both 0 and 1)
    filtered_samples = [
        sample for sample in hf_dataset_cache
        if sample.get("text") and sample["text"].strip()
    ]

    if not filtered_samples:
        raise HTTPException(status_code=400, detail="No valid samples found.")

    texts = [sample["text"] for sample in filtered_samples]
    labels = [sample["label"] for sample in filtered_samples]

    vectorizer = CountVectorizer()
    X = vectorizer.fit_transform(texts)
    y = labels

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

    if model_type == "naive-bayes":
        clf = MultinomialNB()
    elif model_type == "svm":
        clf = SVC()
    else:
        raise HTTPException(status_code=400, detail="Invalid model type (use 'naive-bayes' or 'svm')")

    clf.fit(X_train, y_train)
    y_pred = clf.predict(X_test)

    cm = confusion_matrix(y_test, y_pred, labels=[0, 1])
    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=["Bukan Judi", "Judi"])

    buf = io.BytesIO()
    disp.plot(cmap='Blues')
    plt.tight_layout()
    plt.savefig(buf, format="png")
    plt.close()
    buf.seek(0)
    return StreamingResponse(buf, media_type="image/png")

@app.post("/predict", response_model=PredictionResponse)
async def predict(input_data: PredictionInput):
    try:
        threshold_low = 0.4
        threshold_high = 0.7

        text = str(input_data.text) if input_data.text else ""
        text = emoji.replace_emoji(text, replace='').strip()

        # Tokenize input text
        inputs = tokenizer(
            text,
            return_tensors="pt",
            padding=True,
            truncation=True,
            max_length=128
        )
        inputs = {key: val.to(device) for key, val in inputs.items()}

        # Make prediction
        with torch.no_grad():
            outputs = model(**inputs)
            probs = F.softmax(outputs.logits, dim=-1)
            prob_0 = probs[0][0].item()
            prob_1 = probs[0][1].item()

        # Interpret prediction
        if prob_1 >= threshold_high:
            prediction = 1
            prediction_text = f'judi ({prob_1:.2f})'
            status = "judi"
            confidence = prob_1
        elif prob_1 <= threshold_low:
            prediction = 0
            prediction_text = f'bukan judi ({prob_0:.2f})'
            status = "bukan judi"
            confidence = prob_0
        else:
            prediction = -1
            prediction_text = f'ragu-ragu ({prob_1:.2f})'
            status = "ragu-ragu"
            confidence = max(prob_0, prob_1)

        highlighted_tokens = get_important_tokens(text, tokenizer, model, target_class=1)

        return PredictionResponse(
            prediction=prediction,
            prediction_text=prediction_text,
            probabilities={"prob_bukan_judi": prob_0, "prob_judi": prob_1},
            confidence=confidence,
            status=status,
            highlighted_tokens=highlighted_tokens
        )

    except Exception as e:
        raise HTTPException(status_code=400, detail=f"Prediction error: {str(e)}")

@app.get("/predict")
async def predict_get(request: Request):
    """GET endpoint to render prediction form"""
    return templates.TemplateResponse(
        request=request, name="predict.html", context={}
    )

@app.get("/samples")
async def get_samples():
    file_path = "static/samples.csv"
    text_field = "text"

    # Try loading from local CSV first
    try:
        if os.path.exists(file_path):
            with open(file_path, newline='', encoding='utf-8') as csvfile:
                reader = csv.DictReader(csvfile)
                samples = [row["text"] for row in reader if row.get("text")]
                return samples
    except Exception as e:
        # Log CSV read failure and fallback
        print(f"Failed to read CSV: {e}")

    # Fallback to Hugging Face dataset
    global hf_dataset_cache
    if hf_dataset_cache is not None:
        samples = [example[text_field] for example in hf_dataset_cache if text_field in example]
        return samples
    else:
        raise HTTPException(status_code=500, detail="Hugging Face dataset is not available.")