File size: 2,706 Bytes
e84ab4f
 
 
 
 
 
 
 
 
079f46c
e84ab4f
079f46c
 
e84ab4f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3e7a192
 
 
 
 
 
e84ab4f
079f46c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c775b3d
 
 
 
 
 
079f46c
c775b3d
e84ab4f
c775b3d
079f46c
 
 
 
e84ab4f
079f46c
e84ab4f
079f46c
e84ab4f
 
 
 
 
 
 
 
 
 
 
c775b3d
 
 
 
 
 
 
e84ab4f
c775b3d
079f46c
 
 
e84ab4f
 
 
 
 
c775b3d
 
 
 
 
e84ab4f
 
 
079f46c
e84ab4f
 
c775b3d
 
079f46c
c775b3d
 
e84ab4f
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
import argparse
import logging
import signal
import sys
import time

import schedule

from cleaner import clean_job
from jobstreet import scrape_jobstreet
from linkedin import scrape_linkedin
from normalize import normalize
from supabase_client import supabase

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
)
log = logging.getLogger("scraper")

KEYWORDS = [
    "software engineer",
    "backend developer",
    "frontend developer",
    "data scientist",
    "full stack developer",
    "devops engineer",
    "ui ux designer",
    "mobile developer",
    # Non-IT (untuk variasi dataset di luar teknologi)
    "marketing",
    "accounting",
    "customer service",
    "sales",
    "finance",
]


def push(jobs, source):
    batch = []
    seen = set()

    for job in jobs:
        job = clean_job(job)
        if not job:
            continue

        url = job["url"]
        if url in seen:
            continue
        seen.add(url)

        batch.append(normalize(job, source))

    if not batch:
        return 0

    try:
        supabase.table("jobs") \
            .upsert(batch, on_conflict="url") \
            .execute()
    except Exception as e:
        log.error("[Supabase] Upsert gagal (%s): %s", source, e)
        return 0

    return len(batch)


def scrape_source(scraper, source, keyword):
    try:
        jobs = scraper(keyword)
    except Exception as e:
        log.error("[%s] Scraping gagal (keyword=%s): %s", source, keyword, e)
        return

    inserted = push(jobs, source)
    log.info("[%s] %s: %d ditemukan, %d dimasukkan", source, keyword, len(jobs), inserted)


def main():
    for keyword in KEYWORDS:
        scrape_source(scrape_linkedin, "LinkedIn", keyword)
        scrape_source(scrape_jobstreet, "JobStreet", keyword)


running = True


def shutdown(signum, frame):
    global running
    log.info("Ctrl+C ditekan. Menghentikan scraper...")
    running = False


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Job scraper LinkedIn & JobStreet")
    parser.add_argument("--once", action="store_true",
                        help="Jalankan sekali lalu keluar (untuk cron/CI)")
    args = parser.parse_args()

    signal.signal(signal.SIGINT, shutdown)
    signal.signal(signal.SIGTERM, shutdown)

    main()  # jalankan langsung saat pertama kali start

    if args.once:
        sys.exit(0)

    schedule.every(24).hours.do(main)
    log.info("Job scraper berjalan setiap 24 jam...")
    log.info("Tekan Ctrl+C untuk menghentikan.")

    while running:
        schedule.run_pending()
        time.sleep(60)

    log.info("Scraper dihentikan. Sampai jumpa!")
    sys.exit(0)