inisialisasi projek

.env.example

@@ -0,0 +1,2 @@
+SUPABASE_URL=your_supabase_url_here
+SUPABASE_KEY=your_supabase_anon_key_here

.gitignore

@@ -0,0 +1,9 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.pyc
+
+# Local configuration and secrets
+.env
+
+# VS Code settings
+.vscode/

base.py

@@ -0,0 +1,11 @@
+import random
+import time
+
+def human_delay(min_sec=2, max_sec=6):
+    time.sleep(random.uniform(min_sec, max_sec))
+
+
+def jitter_scroll(page):
+    page.evaluate("""
+        window.scrollTo(0, document.body.scrollHeight / 2);
+    """)

browser.py

@@ -0,0 +1,18 @@
+from playwright.sync_api import sync_playwright
+
+def get_browser():
+    p = sync_playwright().start()
+
+    browser = p.chromium.launch(
+        headless=True,
+        args=[
+            "--disable-blink-features=AutomationControlled"
+        ]
+    )
+
+    context = browser.new_context(
+        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120 Safari/537.36",
+        viewport={"width": 1280, "height": 800}
+    )
+
+    return p, browser, context

cleaner.py

@@ -0,0 +1,20 @@
+def clean_job(job):
+    company = job.get("company")
+    title = job.get("title")
+    url = job.get("url")
+
+    # safety check
+    if not company or not title or not url:
+        return None
+
+    company = company.strip()
+    title = title.strip()
+    url = url.strip()
+
+    if len(company) < 2 or len(title) < 5:
+        return None
+
+    if not url.startswith("http"):
+        return None
+
+    return job

jobstreet.py

@@ -0,0 +1,41 @@
+from bs4 import BeautifulSoup
+from browser import get_browser
+from base import human_delay, jitter_scroll
+
+
+def scrape_jobstreet(keyword="developer"):
+    p, browser, context = get_browser()
+    page = context.new_page()
+
+    url = f"https://www.jobstreet.co.id/id/job-search/{keyword}-jobs/"
+
+    page.goto(url, wait_until="domcontentloaded")
+    human_delay()
+
+    jitter_scroll(page)
+    human_delay()
+
+    soup = BeautifulSoup(page.content(), "html.parser")
+
+    jobs = []
+
+    cards = soup.select("article")
+
+    for card in cards:
+        try:
+            title = card.select_one("a")
+            company = card.select_one("span")
+
+            if title:
+                jobs.append({
+                    "title": title.text.strip(),
+                    "company": company.text.strip() if company else "",
+                    "url": title["href"]
+                })
+        except:
+            continue
+
+    browser.close()
+    p.stop()
+
+    return jobs

linkedin.py

@@ -0,0 +1,42 @@
+from bs4 import BeautifulSoup
+from browser import get_browser
+from base import human_delay, jitter_scroll
+
+
+def scrape_linkedin(keyword="software engineer"):
+    p, browser, context = get_browser()
+    page = context.new_page()
+
+    url = f"https://www.linkedin.com/jobs/search/?keywords={keyword}"
+
+    page.goto(url, wait_until="domcontentloaded")
+    human_delay()
+
+    jitter_scroll(page)
+    human_delay()
+
+    soup = BeautifulSoup(page.content(), "html.parser")
+
+    jobs = []
+
+    cards = soup.select("li")
+
+    for card in cards:
+        try:
+            title = card.select_one("h3")
+            company = card.select_one("h4")
+            link = card.select_one("a")
+
+            if title and company and link:
+                jobs.append({
+                    "title": title.text.strip(),
+                    "company": company.text.strip(),
+                    "url": "https://www.linkedin.com" + link["href"]
+                })
+        except:
+            continue
+
+    browser.close()
+    p.stop()
+
+    return jobs

normalize.py

@@ -0,0 +1,9 @@
+def normalize(job, source):
+    return {
+        "nama_perusahaan": job.get("company"),
+        "posisi_jabatan": job.get("title"),
+        "deskripsi_kualifikasi": job.get("title"),
+        "link_pendaftaran": job.get("url"),
+        "sumber_data": source,
+        "is_active": True
+    }

readme.md

@@ -0,0 +1,59 @@
+# Job Scraper
+
+Proyek ini adalah scraper lowongan kerja yang mengumpulkan data dari LinkedIn dan JobStreet, membersihkannya, lalu menyimpannya ke database Supabase.
+
+## Fitur
+
+- Scraping lowongan dari **LinkedIn** dan **JobStreet**
+- Automasi browser menggunakan Playwright
+- Pembersihan dan normalisasi data sebelum disimpan
+- Penyimpanan data ke Supabase dengan mekanisme upsert
+- Penjadwalan scraping setiap 24 jam
+
+## Setup
+
+1. Clone repository ini
+2. Buat virtual environment (disarankan Python 3.10+):
+
+    python -m venv venv
+    venv\Scripts\activate
+
+3. Install dependensi:
+
+    pip install -r requirements.txt
+
+4. Instal browser untuk Playwright:
+
+    playwright install chromium
+
+5. Buat file `.env` di direktori proyek:
+
+    SUPABASE_URL=your_supabase_url
+    SUPABASE_ANON_KEY=your_supabase_anon_key
+
+6. Jalankan scraper:
+
+    python run.py
+
+## Struktur Modul
+
+- `run.py` - Entry point, menjadwalkan scraping dan menyimpan data
+- `linkedin.py` - Scraping lowongan dari LinkedIn
+- `jobstreet.py` - Scraping lowongan dari JobStreet
+- `browser.py` - Konfigurasi Playwright (Chromium headless)
+- `base.py` - Helper delay dan scroll untuk simulasi manusia
+- `normalize.py` - Normalisasi format data sebelum disimpan
+- `cleaner.py` - Validasi dan pembersihan data yang tidak valid
+- `supabase_client.py` - Koneksi ke Supabase
+
+## Database
+
+Pastikan tabel `lowongan_kerja` sudah ada di Supabase dengan kolom:
+- `nama_perusahaan`
+- `posisi_jabatan`
+- `deskripsi_kualifikasi`
+- `link_pendaftaran`
+- `sumber_data`
+- `is_active`
+
+Kolom `link_pendaftaran` digunakan sebagai konflik untuk upsert.

requirements.txt

@@ -0,0 +1,5 @@
+beautifulsoup4==4.12.3
+playwright==1.40.0
+python-dotenv==1.0.1
+schedule==1.2.2
+supabase==2.3.4

run.py

@@ -0,0 +1,56 @@
+from linkedin import scrape_linkedin
+from jobstreet import scrape_jobstreet
+from normalize import normalize
+from cleaner import clean_job
+from supabase_client import supabase
+import time
+import schedule
+
+
+def push(jobs, source):
+    batch = []
+    seen = set()
+
+    for job in jobs:
+
+        # CLEANING STEP
+        job = clean_job(job)
+        if not job:
+            continue
+
+        url = job["url"]
+
+        if url in seen:
+            continue
+        seen.add(url)
+
+        batch.append(normalize(job, source))
+
+    if batch:
+        supabase.table("lowongan_kerja") \
+            .upsert(batch, on_conflict="link_pendaftaran") \
+            .execute()
+
+    return len(batch)
+
+
+def main():
+    try:
+        print("Scraping LinkedIn...")
+        linkedin_jobs = scrape_linkedin()
+        print("LinkedIn inserted:", push(linkedin_jobs, "LinkedIn"))
+
+        print("Scraping JobStreet...")
+        jobstreet_jobs = scrape_jobstreet()
+        print("JobStreet inserted:", push(jobstreet_jobs, "JobStreet"))
+    except Exception as e:
+        print(f"Error during scraping: {e}")
+
+
+if __name__ == "__main__":
+    schedule.every(24).hours.do(main)
+    
+    print("Job scraper running every 24 hours...")
+    while True:
+        schedule.run_pending()
+        time.sleep(60)  # Check every minute

supabase_client.py

@@ -0,0 +1,16 @@
+from dotenv import load_dotenv
+import os
+from supabase import create_client
+
+load_dotenv()
+
+def get_supabase():
+    url = os.getenv("SUPABASE_URL")
+    key = os.getenv("SUPABASE_ANON_KEY")
+
+    if not url or not key:
+        raise ValueError("Missing Supabase env")
+
+    return create_client(url, key)
+
+supabase = get_supabase()