Spaces:
Sleeping
Sleeping
File size: 2,706 Bytes
e84ab4f 079f46c e84ab4f 079f46c e84ab4f 3e7a192 e84ab4f 079f46c c775b3d 079f46c c775b3d e84ab4f c775b3d 079f46c e84ab4f 079f46c e84ab4f 079f46c e84ab4f c775b3d e84ab4f c775b3d 079f46c e84ab4f c775b3d e84ab4f 079f46c e84ab4f c775b3d 079f46c c775b3d e84ab4f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 | import argparse
import logging
import signal
import sys
import time
import schedule
from cleaner import clean_job
from jobstreet import scrape_jobstreet
from linkedin import scrape_linkedin
from normalize import normalize
from supabase_client import supabase
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
)
log = logging.getLogger("scraper")
KEYWORDS = [
"software engineer",
"backend developer",
"frontend developer",
"data scientist",
"full stack developer",
"devops engineer",
"ui ux designer",
"mobile developer",
# Non-IT (untuk variasi dataset di luar teknologi)
"marketing",
"accounting",
"customer service",
"sales",
"finance",
]
def push(jobs, source):
batch = []
seen = set()
for job in jobs:
job = clean_job(job)
if not job:
continue
url = job["url"]
if url in seen:
continue
seen.add(url)
batch.append(normalize(job, source))
if not batch:
return 0
try:
supabase.table("jobs") \
.upsert(batch, on_conflict="url") \
.execute()
except Exception as e:
log.error("[Supabase] Upsert gagal (%s): %s", source, e)
return 0
return len(batch)
def scrape_source(scraper, source, keyword):
try:
jobs = scraper(keyword)
except Exception as e:
log.error("[%s] Scraping gagal (keyword=%s): %s", source, keyword, e)
return
inserted = push(jobs, source)
log.info("[%s] %s: %d ditemukan, %d dimasukkan", source, keyword, len(jobs), inserted)
def main():
for keyword in KEYWORDS:
scrape_source(scrape_linkedin, "LinkedIn", keyword)
scrape_source(scrape_jobstreet, "JobStreet", keyword)
running = True
def shutdown(signum, frame):
global running
log.info("Ctrl+C ditekan. Menghentikan scraper...")
running = False
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Job scraper LinkedIn & JobStreet")
parser.add_argument("--once", action="store_true",
help="Jalankan sekali lalu keluar (untuk cron/CI)")
args = parser.parse_args()
signal.signal(signal.SIGINT, shutdown)
signal.signal(signal.SIGTERM, shutdown)
main() # jalankan langsung saat pertama kali start
if args.once:
sys.exit(0)
schedule.every(24).hours.do(main)
log.info("Job scraper berjalan setiap 24 jam...")
log.info("Tekan Ctrl+C untuk menghentikan.")
while running:
schedule.run_pending()
time.sleep(60)
log.info("Scraper dihentikan. Sampai jumpa!")
sys.exit(0) |