Spaces:
Sleeping
Sleeping
| """ | |
| app.py — Web UI 1-klik untuk menjalankan scraper LinkedIn & JobStreet. | |
| Jalankan: python app.py | |
| Buka: http://localhost:5000 | |
| """ | |
| import logging | |
| import queue | |
| import threading | |
| import time | |
| from datetime import datetime | |
| from flask import Flask, jsonify, render_template, request | |
| # Impor fungsi scraping tanpa trigger scheduler di run.py | |
| import run as scraper_run | |
| from supabase_client import supabase | |
| app = Flask(__name__) | |
| # ── State global ────────────────────────────────────────────── | |
| state = { | |
| "running": False, | |
| "last_run": None, | |
| "last_result": None, # {started_at, finished_at, duration, logs} | |
| "stats": {"total_jobs": 0, "last_inserted": 0}, | |
| } | |
| log_queue: queue.Queue = queue.Queue() | |
| logs: list[str] = [] # buffer max 500 baris | |
| MAX_LOGS = 500 | |
| lock = threading.Lock() | |
| # ── Logging handler yang mengalir ke UI ────────────────────── | |
| class QueueHandler(logging.Handler): | |
| def emit(self, record): | |
| msg = self.format(record) | |
| with lock: | |
| logs.append(msg) | |
| if len(logs) > MAX_LOGS: | |
| logs.pop(0) | |
| try: | |
| log_queue.put_nowait(msg) | |
| except queue.Full: | |
| pass | |
| # Pasang handler ke logger scraper + root | |
| handler = QueueHandler() | |
| handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s", datefmt="%H:%M:%S")) | |
| for name in ("scraper", "linkedin", "jobstreet", ""): | |
| lg = logging.getLogger(name) | |
| lg.addHandler(handler) | |
| lg.setLevel(logging.INFO) | |
| log = logging.getLogger("app") | |
| def _run_scrape_task(keywords: list[str] | None): | |
| """Jalan di background thread. keywords=None → pakai KEYWORDS default.""" | |
| target_kw = keywords if keywords else scraper_run.KEYWORDS | |
| start = time.time() | |
| started_at = datetime.now().isoformat(timespec="seconds") | |
| with lock: | |
| state["running"] = True | |
| state["last_run"] = started_at | |
| logs.clear() | |
| # kosongkan queue lama | |
| while not log_queue.empty(): | |
| try: | |
| log_queue.get_nowait() | |
| except queue.Empty: | |
| break | |
| log.info("=== Scraping dimulai: %d keyword ===", len(target_kw)) | |
| total_inserted = 0 | |
| total_found = 0 | |
| try: | |
| for kw in target_kw: | |
| log.info("→ Keyword: %s", kw) | |
| try: | |
| jobs_li = scraper_run.scrape_linkedin(kw) | |
| ins = scraper_run.push(jobs_li, "LinkedIn") | |
| total_inserted += ins | |
| total_found += len(jobs_li) | |
| log.info("[LinkedIn] %s: %d ditemukan, %d dimasukkan", kw, len(jobs_li), ins) | |
| except Exception as e: | |
| log.error("[LinkedIn] gagal %s: %s", kw, e) | |
| # JobStreet | |
| try: | |
| jobs_js = scraper_run.scrape_jobstreet(kw) | |
| ins = scraper_run.push(jobs_js, "JobStreet") | |
| total_inserted += ins | |
| total_found += len(jobs_js) | |
| log.info("[JobStreet] %s: %d ditemukan, %d dimasukkan", kw, len(jobs_js), ins) | |
| except Exception as e: | |
| log.error("[JobStreet] gagal %s: %s", kw, e) | |
| duration = round(time.time() - start, 1) | |
| log.info("=== Selesai: %d ditemukan, %d dimasukkan dalam %ss ===", total_found, total_inserted, duration) | |
| with lock: | |
| state["last_result"] = { | |
| "started_at": started_at, | |
| "finished_at": datetime.now().isoformat(timespec="seconds"), | |
| "duration_sec": duration, | |
| "found": total_found, | |
| "inserted": total_inserted, | |
| "keywords": target_kw, | |
| } | |
| state["stats"]["last_inserted"] = total_inserted | |
| # refresh total dari DB | |
| try: | |
| res = supabase.table("jobs").select("id", count="exact").limit(1).execute() | |
| state["stats"]["total_jobs"] = res.count or 0 | |
| except Exception: | |
| pass | |
| except Exception as e: | |
| log.error("Scraping error fatal: %s", e, exc_info=True) | |
| finally: | |
| with lock: | |
| state["running"] = False | |
| # ── Routes ──────────────────────────────────────────────────── | |
| def index(): | |
| return render_template("index.html", keywords=scraper_run.KEYWORDS) | |
| def api_keywords(): | |
| return jsonify(scraper_run.KEYWORDS) | |
| def api_status(): | |
| with lock: | |
| return jsonify({ | |
| "running": state["running"], | |
| "last_run": state["last_run"], | |
| "last_result": state["last_result"], | |
| "stats": state["stats"], | |
| "logs": list(logs), # kirim buffer | |
| }) | |
| def api_logs(): | |
| """Polling incremental: ?after=<jumlah_log_yang_sudah_diterima>""" | |
| try: | |
| after = int(request.args.get("after", 0)) | |
| except ValueError: | |
| after = 0 | |
| with lock: | |
| new = logs[after:] | |
| return jsonify({"logs": new, "total": len(logs), "running": state["running"]}) | |
| def api_jobs(): | |
| """Ambil 20 jobs terbaru untuk preview tabel.""" | |
| try: | |
| limit = min(int(request.args.get("limit", 20)), 100) | |
| except ValueError: | |
| limit = 20 | |
| try: | |
| res = supabase.table("jobs").select("*").order("created_at", desc=True).limit(limit).execute() | |
| # update stats juga | |
| with lock: | |
| if res.count is not None: | |
| state["stats"]["total_jobs"] = res.count | |
| return jsonify({"jobs": res.data, "count": len(res.data)}) | |
| except Exception as e: | |
| return jsonify({"error": str(e), "jobs": []}), 500 | |
| def api_stats(): | |
| try: | |
| total = supabase.table("jobs").select("id", count="exact").limit(1).execute() | |
| li = supabase.table("jobs").select("id", count="exact").eq("source", "LinkedIn").limit(1).execute() | |
| js = supabase.table("jobs").select("id", count="exact").eq("source", "JobStreet").limit(1).execute() | |
| return jsonify({ | |
| "total": total.count or 0, | |
| "linkedin": li.count or 0, | |
| "jobstreet": js.count or 0, | |
| }) | |
| except Exception as e: | |
| return jsonify({"error": str(e)}), 500 | |
| def api_run(): | |
| with lock: | |
| if state["running"]: | |
| return jsonify({"error": "Scraper sedang berjalan. Tunggu selesai."}), 409 | |
| data = request.get_json(silent=True) or {} | |
| keywords = data.get("keywords") # optional list[str] | |
| if keywords is not None: | |
| if not isinstance(keywords, list) or not all(isinstance(k, str) and k.strip() for k in keywords): | |
| return jsonify({"error": "keywords harus list string"}), 400 | |
| keywords = [k.strip() for k in keywords if k.strip()] | |
| if not keywords: | |
| return jsonify({"error": "keywords tidak boleh kosong"}), 400 | |
| if len(keywords) > 2: | |
| return jsonify({"error": "maksimal 2 keyword (limit Render Free 512MB)"}), 400 | |
| thread = threading.Thread(target=_run_scrape_task, args=(keywords,), daemon=True) | |
| thread.start() | |
| return jsonify({"status": "started", "keywords": keywords or scraper_run.KEYWORDS}) | |
| def health(): | |
| return jsonify({"ok": True}) | |
| if __name__ == "__main__": | |
| import os | |
| # Pre-load stats | |
| try: | |
| r = supabase.table("jobs").select("id", count="exact").limit(1).execute() | |
| state["stats"]["total_jobs"] = r.count or 0 | |
| except Exception: | |
| pass | |
| port = int(os.getenv("PORT", "5000")) | |
| print(f"→ Buka http://localhost:{port} di browser, klik 'Mulai Crawling'") | |
| app.run(host="0.0.0.0", port=port, debug=True, use_reloader=False, threaded=True) | |