Upload app.py

app.py

@@ -0,0 +1,565 @@
+import os
+import re
+import urllib.request
+import xml.etree.ElementTree as ET
+from email.utils import parsedate_to_datetime
+from urllib.parse import quote
+
+import gradio as gr
+import pandas as pd
+import torch
+from peft import PeftModel
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+# ================================================================
+# CONFIG
+# ================================================================
+
+BASE_MODEL = "distilbert-base-uncased"
+ADAPTER_PATH = "."
+MAX_LENGTH = 128
+BATCH_SIZE = 32
+
+LABELS = {0: "negative", 1: "neutral", 2: "positive"}
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# ================================================================
+# MODEL LOADING (DistilBERT + LoRA)
+# ================================================================
+
+print("Device:", DEVICE)
+
+tokenizer = AutoTokenizer.from_pretrained(ADAPTER_PATH, use_fast=True)
+
+base_model = AutoModelForSequenceClassification.from_pretrained(
+    BASE_MODEL,
+    num_labels=3,
+    id2label=LABELS,
+    label2id={v: k for k, v in LABELS.items()},
+    ignore_mismatched_sizes=True,
+)
+
+model = PeftModel.from_pretrained(base_model, ADAPTER_PATH)
+try:
+    model = model.merge_and_unload()
+except Exception as e:
+    print("Merge skipped:", e)
+
+model = model.to(DEVICE)
+model.eval()
+print("Model ready.")
+
+
+# ================================================================
+# SENTIMENT (batch)
+# ================================================================
+
+def predict_batch(texts):
+    """Her metin için (p_neg, p_neu, p_pos) döndürür."""
+    out = []
+    for i in range(0, len(texts), BATCH_SIZE):
+        chunk = texts[i:i + BATCH_SIZE]
+        inputs = tokenizer(
+            chunk,
+            return_tensors="pt",
+            truncation=True,
+            max_length=MAX_LENGTH,
+            padding=True,
+        )
+        inputs = {k: v.to(DEVICE) for k, v in inputs.items()}
+        with torch.inference_mode():
+            probs = torch.softmax(model(**inputs).logits, dim=-1).cpu().tolist()
+        out.extend(probs)
+    return out
+
+
+# ================================================================
+# RISK KURALLARI (anahtar kelime tabanlı sinyaller)
+# ================================================================
+
+RISK_SIGNALS = {
+    "Hasar / Claim": [
+        "claim", "claims", "payout", "payouts", "loss", "losses",
+        "combined ratio", "reserve", "reserves",
+    ],
+    "Doğal afet": [
+        "hurricane", "earthquake", "flood", "flooding", "wildfire", "storm",
+        "catastrophe", "cyclone", "typhoon", "tornado", "hailstorm", "drought",
+    ],
+    "Büyük kaza": [
+        "crash", "explosion", "collapse", "derailment", "blast", "accident",
+        "fire", "sinking", "spill",
+    ],
+    "Dava / Fraud": [
+        "lawsuit", "sued", "litigation", "fraud", "fine", "fined", "penalty",
+        "investigation", "probe", "settlement", "class action", "regulator",
+    ],
+    "Finansal stres": [
+        "downgrade", "downgraded", "profit warning", "solvency", "writedown",
+        "write-down", "impairment", "insolvency", "bankruptcy", "capital shortfall",
+        "negative outlook", "slump", "plunge",
+    ],
+    "Siber / Operasyonel": [
+        "cyberattack", "cyber attack", "data breach", "breach", "ransomware",
+        "outage",
+    ],
+}
+
+TOPICS = {
+    "Motor insurance": ["motor", "auto insurance", "car insurance", "vehicle", "automobile"],
+    "Health insurance": ["health insurance", "health", "medical", "hospital"],
+    "Catastrophe risk": ["catastrophe", "hurricane", "earthquake", "flood", "wildfire",
+                         "storm", "cyclone", "typhoon", "tornado", "hailstorm"],
+    "Life insurance": ["life insurance", "life insurer", "annuity", "annuities", "pension", "mortality"],
+    "Reinsurance": ["reinsur", "retrocession", "cat bond", "catastrophe bond"],
+    "Property": ["property insurance", "homeowner", "home insurance", "real estate", "fire"],
+    "Cyber": ["cyber", "ransomware", "data breach"],
+}
+
+COMPANIES = [
+    "Allianz", "AXA", "Zurich", "Generali", "Aviva", "Munich Re", "Swiss Re",
+    "Hannover Re", "Chubb", "AIG", "MetLife", "Prudential", "Allstate",
+    "Progressive", "State Farm", "Mapfre", "Ageas", "Lloyd's", "Travelers",
+    "Liberty Mutual", "Tokio Marine", "Ping An",
+]
+
+
+def _compile(words):
+    return re.compile(r"\b(?:%s)" % "|".join(re.escape(w) for w in words), re.I)
+
+
+SIGNAL_RE = {cat: _compile(words) for cat, words in RISK_SIGNALS.items()}
+TOPIC_RE = {t: _compile(words) for t, words in TOPICS.items()}
+COMPANY_RE = {c: re.compile(r"\b%s\b" % re.escape(c), re.I) for c in COMPANIES}
+
+
+def find_signals(text):
+    hits = []
+    for cat, rx in SIGNAL_RE.items():
+        m = rx.findall(text)
+        if m:
+            hits.append((cat, sorted({x.lower() for x in m})[:2]))
+    return hits
+
+
+def find_topics(text):
+    return [t for t, rx in TOPIC_RE.items() if rx.search(text)]
+
+
+def find_companies(text):
+    return [c for c, rx in COMPANY_RE.items() if rx.search(text)]
+
+
+def risk_level(score, high=60, medium=35):
+    if score >= high:
+        return "🔴 HIGH"
+    if score >= medium:
+        return "🟠 MEDIUM"
+    return "🟢 LOW"
+
+
+# ================================================================
+# TEK HABER ANALİZİ
+# ================================================================
+
+def score_item(text, probs):
+    p_neg, p_neu, p_pos = probs
+    sentiment = p_pos - p_neg                       # -1 .. +1
+    signals = find_signals(text)
+    n_sig = len(signals)
+    # Risk indeksi: %65 negatif olasılık + %35 risk sinyali yoğunluğu
+    risk = 100 * (0.65 * p_neg + 0.35 * min(n_sig, 3) / 3)
+    label = LABELS[max(range(3), key=lambda i: probs[i])]
+    reason = "; ".join(f"{c} ({', '.join(k)})" for c, k in signals) or "-"
+    return {
+        "label": label,
+        "sentiment": round(sentiment, 3),
+        "signals": n_sig,
+        "risk_score": round(risk, 1),
+        "risk": risk_level(risk),
+        "reason": reason,
+    }
+
+
+def analyze_news(text):
+    text = (text or "").strip()
+    if not text:
+        return "Lütfen bir sigorta haber başlığı girin.", "", ""
+
+    probs = predict_batch([text])[0]
+    s = score_item(text, probs)
+
+    companies = find_companies(text) or ["-"]
+    topics = find_topics(text) or ["-"]
+
+    result = (
+        f"Sentiment: {s['label'].upper()}\n"
+        f"Skor: {s['sentiment']:+.2f}\n"
+        f"Güven: {max(probs) * 100:.1f}%"
+    )
+    prob_text = (
+        f"Negative: {probs[0] * 100:.1f}%\n"
+        f"Neutral: {probs[1] * 100:.1f}%\n"
+        f"Positive: {probs[2] * 100:.1f}%"
+    )
+    risk_text = (
+        f"Şirket: {', '.join(companies)}\n"
+        f"Risk: {s['risk']} ({s['risk_score']}/100)\n"
+        f"Risk sinyali: {s['signals']}\n"
+        f"Neden: {s['reason']}\n"
+        f"Konu: {', '.join(topics)}"
+    )
+    return result, prob_text, risk_text
+
+
+# ================================================================
+# TOPLU ANALİZ PIPELINE
+# ================================================================
+
+def analyze_items(items):
+    """items: [{'headline':str, 'date':datetime|None, 'company':str|None}]"""
+    if not items:
+        return None, None, None
+
+    texts = [it["headline"] for it in items]
+    probs = predict_batch(texts)
+
+    rows = []
+    for it, p in zip(items, probs):
+        s = score_item(it["headline"], p)
+        companies = [it["company"]] if it.get("company") else find_companies(it["headline"])
+        rows.append({
+            "date": it.get("date"),
+            "headline": it["headline"],
+            "companies": companies or ["(Sektör)"],
+            "topics": find_topics(it["headline"]),
+            **s,
+        })
+    df = pd.DataFrame(rows)
+
+    has_dates = df["date"].notna().sum() >= 4
+    if has_dates:
+        df["date"] = pd.to_datetime(df["date"], utc=True, errors="coerce")
+        mid = df["date"].dropna().sort_values().iloc[len(df["date"].dropna()) // 2]
+        df["recent"] = df["date"] >= mid
+    else:
+        df["recent"] = False
+
+    # ---- Şirket tablosu ----
+    ex = df.explode("companies")
+    comp_rows = []
+    for comp, g in ex.groupby("companies"):
+        risk = g["risk_score"].mean()
+        trend = "—"
+        if has_dates:
+            r, e = g[g["recent"]], g[~g["recent"]]
+            if len(r) and len(e):
+                d = r["sentiment"].mean() - e["sentiment"].mean()
+                trend = ("Improving ↑" if d > 0.15
+                         else "Deteriorating ↓" if d < -0.15 else "Stable →")
+        comp_rows.append({
+            "Company": comp,
+            "News": len(g),
+            "Avg sentiment": round(g["sentiment"].mean(), 2),
+            "Risk signals": int(g["signals"].sum()),
+            "High-risk news": int((g["risk_score"] >= 60).sum()),
+            "Risk score": round(risk, 1),
+            "Risk": risk_level(risk, high=55, medium=30),
+            "Trend": trend,
+        })
+    comp_df = pd.DataFrame(comp_rows).sort_values("Risk score", ascending=False)
+
+    # ---- Konu / trend tablosu ----
+    topic_rows = []
+    n_recent = max(int(df["recent"].sum()), 1)
+    n_early = max(int((~df["recent"]).sum()), 1)
+    for topic in TOPICS:
+        mask = df["topics"].apply(lambda t: topic in t)
+        if not mask.any():
+            continue
+        arrow = "—"
+        if has_dates:
+            share_r = (mask & df["recent"]).sum() / n_recent
+            share_e = (mask & ~df["recent"]).sum() / n_early
+            if share_e == 0:
+                arrow = "↑↑"
+            else:
+                ratio = share_r / share_e
+                arrow = ("↑↑" if ratio >= 1.5 else "↑" if ratio >= 1.15
+                         else "↓" if ratio <= 0.8 else "→")
+        topic_rows.append({
+            "Topic": topic,
+            "News": int(mask.sum()),
+            "Avg sentiment": round(df.loc[mask, "sentiment"].mean(), 2),
+            "Trend": arrow,
+        })
+    topic_df = pd.DataFrame(topic_rows).sort_values("News", ascending=False) \
+        if topic_rows else pd.DataFrame(columns=["Topic", "News", "Avg sentiment", "Trend"])
+
+    # ---- Haber tablosu ----
+    news_df = df.assign(
+        companies=df["companies"].apply(", ".join),
+        topics=df["topics"].apply(", ".join),
+        date=df["date"].apply(lambda d: d.strftime("%Y-%m-%d") if pd.notna(d) else ""),
+    )[["date", "companies", "headline", "label", "sentiment", "risk", "risk_score", "reason", "topics"]]
+    news_df = news_df.sort_values("risk_score", ascending=False)
+
+    return news_df, comp_df, topic_df
+
+
+# ================================================================
+# TOPLU GİRİŞ (metin / CSV)
+# ================================================================
+
+def _pick(cols, names):
+    for n in names:
+        if n in cols:
+            return n
+    return None
+
+
+def batch_analyze(text, file):
+    items = []
+
+    path = file if isinstance(file, str) else getattr(file, "name", None)
+    if path:
+        df = pd.read_csv(path)
+        df.columns = [c.strip().lower() for c in df.columns]
+        h = _pick(df.columns, ["headline", "title", "text", "news", "haber", "baslik"])
+        d = _pick(df.columns, ["date", "published", "pubdate", "tarih"])
+        c = _pick(df.columns, ["company", "sirket", "şirket"])
+        if h is None:
+            raise gr.Error("CSV'de 'headline' (veya title/text/news) kolonu bulunamadı.")
+        for _, r in df.iterrows():
+            if pd.isna(r[h]) or not str(r[h]).strip():
+                continue
+            items.append({
+                "headline": str(r[h]).strip(),
+                "date": pd.to_datetime(r[d], utc=True, errors="coerce") if d else None,
+                "company": str(r[c]).strip() if c and pd.notna(r[c]) else None,
+            })
+
+    for line in (text or "").splitlines():
+        line = line.strip()
+        if line:
+            items.append({"headline": line, "date": None, "company": None})
+
+    if not items:
+        raise gr.Error("Haber girin (her satıra bir başlık) veya CSV yükleyin.")
+    return analyze_items(items)
+
+
+# ================================================================
+# CANLI TARAMA (Google News RSS)
+# ================================================================
+
+def fetch_news(company, days, limit):
+    q = quote(f'"{company}" insurance when:{int(days)}d')
+    url = f"https://news.google.com/rss/search?q={q}&hl=en-US&gl=US&ceid=US:en"
+    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
+    with urllib.request.urlopen(req, timeout=15) as resp:
+        root = ET.fromstring(resp.read())
+
+    items = []
+    for it in root.iter("item"):
+        title = (it.findtext("title") or "").strip()
+        title = re.sub(r"\s+-\s+[^-]+$", "", title)    # " - Kaynak" kısmını at
+        try:
+            date = parsedate_to_datetime(it.findtext("pubDate"))
+        except Exception:
+            date = None
+        if title:
+            items.append({"headline": title, "date": date, "company": company})
+        if len(items) >= limit:
+            break
+    return items
+
+
+def live_scan(company_text, days, limit):
+    companies = [c.strip() for c in (company_text or "").split(",") if c.strip()]
+    if not companies:
+        raise gr.Error("En az bir şirket adı girin.")
+
+    items, errors = [], []
+    for c in companies:
+        try:
+            items += fetch_news(c, days, int(limit))
+        except Exception as e:
+            errors.append(f"{c}: {e}")
+
+    if not items:
+        raise gr.Error("Haber alınamadı. İnternet erişimini kontrol edin. " + " | ".join(errors))
+    return analyze_items(items)
+
+
+# ================================================================
+# GRADIO UI
+# ================================================================
+
+EXAMPLES = [
+    ["Insurance company reports record premium growth"],
+    ["Allianz faces major claims increase after severe floods"],
+    ["Zurich maintained stable operations in the third quarter."],
+    ["AXA downgraded after reserve shortfall and fraud investigation"],
+    ["Hurricane losses push reinsurance rates higher"],
+]
+
+BATCH_PLACEHOLDER = (
+    "Allianz reports record premium growth\n"
+    "Generali hit by wildfire claims surge\n"
+    "Zurich faces lawsuit over auto claims"
+)
+
+NEWS_COLS = ["date", "companies", "headline", "label", "sentiment", "risk", "risk_score", "reason", "topics"]
+
+BRAND = "Goshawk_insurance_bert"
+
+LOGO_SVG = """
+<svg width="64" height="64" viewBox="0 0 64 64" xmlns="http://www.w3.org/2000/svg" aria-label="Goshawk logo">
+  <defs>
+    <linearGradient id="gs" x1="0" y1="0" x2="0" y2="1">
+      <stop offset="0" stop-color="#1E4F9C"/><stop offset="1" stop-color="#0B1F3A"/>
+    </linearGradient>
+  </defs>
+  <path d="M32 3 L57 12 V32 C57 47 46 57 32 61 C18 57 7 47 7 32 V12 Z"
+        fill="url(#gs)" stroke="#F2B134" stroke-width="2.5" stroke-linejoin="round"/>
+  <path d="M15 37 C17 25 27 16 40 18 L51 24 L42 27 C45 32 43 38 38 41 L32 51 L28 43 C22 43 17 41 15 37 Z"
+        fill="#FFFFFF"/>
+  <path d="M51 24 L42 27 L45 33 Z" fill="#F2B134"/>
+  <path d="M22 27 L36 23 L35 26 L24 30 Z" fill="#0B1F3A" opacity=".85"/>
+  <circle cx="35" cy="29" r="2.2" fill="#F2B134"/>
+  <circle cx="35.4" cy="29" r="1" fill="#0B1F3A"/>
+</svg>
+"""
+
+CSS = """
+.gradio-container {max-width: 1100px !important; margin: auto;
+  font-family: -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;}
+#brand-header {display:flex; align-items:center; gap:16px; padding:20px 22px;
+  border-radius:18px; color:#fff; margin-bottom:10px;
+  background: linear-gradient(135deg, #0B1F3A 0%, #1E4F9C 70%, #2B6CC4 100%);
+  box-shadow: 0 6px 20px rgba(11,31,58,.25);}
+#brand-header .logo {flex:0 0 auto; filter: drop-shadow(0 2px 6px rgba(0,0,0,.35));}
+#brand-header .title {font-size:1.55rem; font-weight:800; letter-spacing:.2px; line-height:1.15;}
+#brand-header .title span {color:#F2B134;}
+#brand-header .sub {opacity:.85; font-size:.92rem; margin-top:4px;}
+#brand-header .badges {margin-top:10px; display:flex; flex-wrap:wrap; gap:6px;}
+#brand-header .badge {background:rgba(255,255,255,.14); border:1px solid rgba(255,255,255,.25);
+  padding:3px 10px; border-radius:999px; font-size:.74rem; backdrop-filter: blur(4px);}
+.legend {display:flex; flex-wrap:wrap; gap:8px; margin:4px 0 12px;}
+.legend .chip {padding:4px 12px; border-radius:999px; font-size:.8rem; font-weight:600;
+  border:1px solid rgba(128,128,128,.3);}
+.chip.high {background:#fde8e8; color:#a31212;}
+.chip.med {background:#fff1dc; color:#9a5a00;}
+.chip.low {background:#e3f6e8; color:#14702d;}
+button.primary {background: linear-gradient(135deg,#1E4F9C,#0B1F3A) !important;
+  border:none !important; color:#fff !important; font-weight:700 !important;
+  border-radius:12px !important;}
+button.primary:hover {filter: brightness(1.15);}
+.tab-nav button, button[role="tab"] {font-weight:600 !important;}
+textarea, input[type="text"] {border-radius:10px !important;}
+#footer {text-align:center; font-size:.82rem; opacity:.75; padding:10px 0 4px;}
+@media (max-width: 640px) {
+  #brand-header {padding:14px; gap:12px;}
+  #brand-header .title {font-size:1.2rem;}
+  #brand-header .logo svg {width:48px; height:48px;}
+}
+"""
+
+HEADER_HTML = f"""
+<div id="brand-header">
+  <div class="logo">{LOGO_SVG}</div>
+  <div>
+    <div class="title">Goshawk<span>_</span>insurance<span>_</span>bert</div>
+    <div class="sub">Sigorta haberleri için sentiment, risk sinyali ve erken uyarı sistemi</div>
+    <div class="badges">
+      <span class="badge">DistilBERT + LoRA</span>
+      <span class="badge">Macro F1 78.06%</span>
+      <span class="badge">Early Warning</span>
+      <span class="badge">EN headlines</span>
+    </div>
+  </div>
+</div>
+<div class="legend">
+  <span class="chip high">🔴 HIGH RISK</span>
+  <span class="chip med">🟠 MEDIUM RISK</span>
+  <span class="chip low">🟢 LOW RISK</span>
+</div>
+"""
+
+with gr.Blocks(
+    title=BRAND,
+    theme=gr.themes.Soft(primary_hue="blue", secondary_hue="amber", neutral_hue="slate"),
+    css=CSS,
+) as demo:
+
+    gr.HTML(HEADER_HTML)
+
+    # ---------------- TAB 1 ----------------
+    with gr.Tab("📰 Tek Haber"):
+        with gr.Row():
+            with gr.Column():
+                text_input = gr.Textbox(label="Insurance News", lines=4,
+                                        placeholder="Allianz faces major claims increase...")
+                btn = gr.Button("🔍 Analiz Et", variant="primary")
+            with gr.Column():
+                out_pred = gr.Textbox(label="Sentiment", lines=3)
+                out_prob = gr.Textbox(label="Olasılıklar", lines=3)
+                out_risk = gr.Textbox(label="Risk Raporu", lines=5)
+        outs = [out_pred, out_prob, out_risk]
+        btn.click(analyze_news, text_input, outs)
+        text_input.submit(analyze_news, text_input, outs)
+        gr.Examples(EXAMPLES, inputs=text_input)
+
+    # ---------------- TAB 2 ----------------
+    with gr.Tab("📚 Toplu Analiz"):
+        gr.Markdown(
+            "Her satıra bir başlık yazın **veya** CSV yükleyin "
+            "(kolonlar: `headline`, opsiyonel `date`, `company`). "
+            "Tarih varsa **trend** hesaplanır."
+        )
+        b_text = gr.Textbox(label="Başlıklar (satır başına bir haber)", lines=6,
+                            placeholder=BATCH_PLACEHOLDER)
+        b_file = gr.File(label="CSV (opsiyonel)", file_types=[".csv"])
+        b_btn = gr.Button("🚀 Toplu Analiz", variant="primary")
+        gr.Markdown("### 🏢 Şirket Risk Tablosu (Underwriting / Rakip İstihbaratı)")
+        b_comp = gr.Dataframe(wrap=True)
+        gr.Markdown("### 📈 Sektör Konu Trendi")
+        b_topic = gr.Dataframe(wrap=True)
+        gr.Markdown("### 🗞️ Haber Detayı (riskten düşüğe)")
+        b_news = gr.Dataframe(wrap=True)
+        b_btn.click(batch_analyze, [b_text, b_file], [b_news, b_comp, b_topic])
+
+    # ---------------- TAB 3 ----------------
+    with gr.Tab("🌐 Canlı Tarama"):
+        gr.Markdown(
+            "Google News RSS'ten şirket haberlerini çekip otomatik analiz eder "
+            "(sunucunun internet erişimi gerekir)."
+        )
+        l_comp = gr.Textbox(label="Şirketler (virgülle)", value="Allianz, AXA, Zurich, Generali")
+        with gr.Row():
+            l_days = gr.Slider(7, 90, value=30, step=1, label="Son kaç gün")
+            l_limit = gr.Slider(5, 50, value=20, step=1, label="Şirket başına haber")
+        l_btn = gr.Button("🔎 Haberleri Tara", variant="primary")
+        gr.Markdown("### 🏢 Rakip Karşılaştırma")
+        l_compdf = gr.Dataframe(wrap=True)
+        gr.Markdown("### 📈 Konu Trendi")
+        l_topic = gr.Dataframe(wrap=True)
+        gr.Markdown("### 🗞️ Haberler")
+        l_news = gr.Dataframe(wrap=True)
+        l_btn.click(live_scan, [l_comp, l_days, l_limit], [l_news, l_compdf, l_topic])
+
+    gr.HTML('<div id="footer">🦅 <b>Goshawk_insurance_bert</b> · Karar destek aracıdır</div>')
+    gr.Markdown(
+        "**Not:** Risk skoru = %65 model negatif olasılığı + %35 anahtar-kelime risk sinyali. "
+        "Karar destek amaçlıdır; tek başına underwriting kararı için kullanılmamalıdır."
+    )
+
+
+if __name__ == "__main__":
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=int(os.environ.get("PORT", 7860)),
+        show_error=True,
+    )