Blame for app.py

"""
app.py — Web UI 1-klik untuk menjalankan scraper LinkedIn & JobStreet.
Jalankan:  python app.py
Buka:      http://localhost:5000
"""
import logging
import queue
import threading
import time
from datetime import datetime

from flask import Flask, jsonify, render_template, request

# Impor fungsi scraping tanpa trigger scheduler di run.py
import run as scraper_run
from supabase_client import supabase

app = Flask(__name__)

# ── State global ──────────────────────────────────────────────
state = {
    "running": False,
    "last_run": None,
    "last_result": None,  # {started_at, finished_at, duration, logs}
    "stats": {"total_jobs": 0, "last_inserted": 0},
}
log_queue: queue.Queue = queue.Queue()
logs: list[str] = []  # buffer max 500 baris
MAX_LOGS = 500
lock = threading.Lock()


# ── Logging handler yang mengalir ke UI ──────────────────────
class QueueHandler(logging.Handler):
    def emit(self, record):
        msg = self.format(record)
        with lock:
            logs.append(msg)
            if len(logs) > MAX_LOGS:
                logs.pop(0)
        try:
            log_queue.put_nowait(msg)
        except queue.Full:
            pass


# Pasang handler ke logger scraper + root
handler = QueueHandler()
handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s", datefmt="%H:%M:%S"))
for name in ("scraper", "linkedin", "jobstreet", ""):
    lg = logging.getLogger(name)
    lg.addHandler(handler)
    lg.setLevel(logging.INFO)

log = logging.getLogger("app")


def _run_scrape_task(keywords: list[str] | None):
    """Jalan di background thread. keywords=None → pakai KEYWORDS default."""
    target_kw = keywords if keywords else scraper_run.KEYWORDS
    start = time.time()
    started_at = datetime.now().isoformat(timespec="seconds")
    with lock:
        state["running"] = True
        state["last_run"] = started_at
        logs.clear()
        # kosongkan queue lama
        while not log_queue.empty():
            try:
                log_queue.get_nowait()
            except queue.Empty:
                break

    log.info("=== Scraping dimulai: %d keyword ===", len(target_kw))
    total_inserted = 0
    total_found = 0
    try:
        for kw in target_kw:
            log.info("→ Keyword: %s", kw)
            # LinkedIn
            try:
                jobs_li = scraper_run.scrape_linkedin(kw)
                ins = scraper_run.push(jobs_li, "LinkedIn")
                total_inserted += ins
                total_found += len(jobs_li)
                log.info("[LinkedIn] %s: %d ditemukan, %d dimasukkan", kw, len(jobs_li), ins)
            except Exception as e:
                log.error("[LinkedIn] gagal %s: %s", kw, e)

            # JobStreet
            try:
                jobs_js = scraper_run.scrape_jobstreet(kw)
                ins = scraper_run.push(jobs_js, "JobStreet")
                total_inserted += ins
                total_found += len(jobs_js)
                log.info("[JobStreet] %s: %d ditemukan, %d dimasukkan", kw, len(jobs_js), ins)
            except Exception as e:
                log.error("[JobStreet] gagal %s: %s", kw, e)

        duration = round(time.time() - start, 1)
        log.info("=== Selesai: %d ditemukan, %d dimasukkan dalam %ss ===", total_found, total_inserted, duration)
        with lock:
            state["last_result"] = {
                "started_at": started_at,
                "finished_at": datetime.now().isoformat(timespec="seconds"),
                "duration_sec": duration,
                "found": total_found,
                "inserted": total_inserted,
                "keywords": target_kw,
            }
            state["stats"]["last_inserted"] = total_inserted
            # refresh total dari DB
            try:
                res = supabase.table("jobs").select("id", count="exact").limit(1).execute()
                state["stats"]["total_jobs"] = res.count or 0
            except Exception:
                pass
    except Exception as e:
        log.error("Scraping error fatal: %s", e, exc_info=True)
    finally:
        with lock:
            state["running"] = False


# ── Routes ────────────────────────────────────────────────────
@app.route("/")
def index():
    return render_template("index.html", keywords=scraper_run.KEYWORDS)


@app.route("/api/keywords")
def api_keywords():
    return jsonify(scraper_run.KEYWORDS)


@app.route("/api/status")
def api_status():
    with lock:
        return jsonify({
            "running": state["running"],
            "last_run": state["last_run"],
            "last_result": state["last_result"],
            "stats": state["stats"],
            "logs": list(logs),  # kirim buffer
        })


@app.route("/api/logs")
def api_logs():
    """Polling incremental: ?after=<jumlah_log_yang_sudah_diterima>"""
    try:
        after = int(request.args.get("after", 0))
    except ValueError:
        after = 0
    with lock:
        new = logs[after:]
        return jsonify({"logs": new, "total": len(logs), "running": state["running"]})


@app.route("/api/jobs")
def api_jobs():
    """Ambil 20 jobs terbaru untuk preview tabel."""
    try:
        limit = min(int(request.args.get("limit", 20)), 100)
    except ValueError:
        limit = 20
    try:
        res = supabase.table("jobs").select("*").order("created_at", desc=True).limit(limit).execute()
        # update stats juga
        with lock:
            if res.count is not None:
                state["stats"]["total_jobs"] = res.count
        return jsonify({"jobs": res.data, "count": len(res.data)})
    except Exception as e:
        return jsonify({"error": str(e), "jobs": []}), 500


@app.route("/api/stats")
def api_stats():
    try:
        total = supabase.table("jobs").select("id", count="exact").limit(1).execute()
        li = supabase.table("jobs").select("id", count="exact").eq("source", "LinkedIn").limit(1).execute()
        js = supabase.table("jobs").select("id", count="exact").eq("source", "JobStreet").limit(1).execute()
        return jsonify({
            "total": total.count or 0,
            "linkedin": li.count or 0,
            "jobstreet": js.count or 0,
        })
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route("/api/run", methods=["POST"])
def api_run():
    with lock:
        if state["running"]:
            return jsonify({"error": "Scraper sedang berjalan. Tunggu selesai."}), 409

    data = request.get_json(silent=True) or {}
    keywords = data.get("keywords")  # optional list[str]
    if keywords is not None:
        if not isinstance(keywords, list) or not all(isinstance(k, str) and k.strip() for k in keywords):
            return jsonify({"error": "keywords harus list string"}), 400
        keywords = [k.strip() for k in keywords if k.strip()]
        if not keywords:
            return jsonify({"error": "keywords tidak boleh kosong"}), 400
        if len(keywords) > 2:
            return jsonify({"error": "maksimal 2 keyword (limit Render Free 512MB)"}), 400

    thread = threading.Thread(target=_run_scrape_task, args=(keywords,), daemon=True)
    thread.start()
    return jsonify({"status": "started", "keywords": keywords or scraper_run.KEYWORDS})


@app.route("/api/health")
def health():
    return jsonify({"ok": True})


if __name__ == "__main__":
    import os
    # Pre-load stats
    try:
        r = supabase.table("jobs").select("id", count="exact").limit(1).execute()
        state["stats"]["total_jobs"] = r.count or 0
    except Exception:
        pass
    port = int(os.getenv("PORT", "5000"))
    print(f"→ Buka http://localhost:{port} di browser, klik 'Mulai Crawling'")
    app.run(host="0.0.0.0", port=port, debug=True, use_reloader=False, threaded=True)