Spaces:
Sleeping
Sleeping
| import argparse | |
| import logging | |
| import signal | |
| import sys | |
| import time | |
| import schedule | |
| from cleaner import clean_job | |
| from jobstreet import scrape_jobstreet | |
| from linkedin import scrape_linkedin | |
| from normalize import normalize | |
| from supabase_client import supabase | |
| logging.basicConfig( | |
| level=logging.INFO, | |
| format="%(asctime)s [%(levelname)s] %(message)s", | |
| ) | |
| log = logging.getLogger("scraper") | |
| KEYWORDS = [ | |
| "software engineer", | |
| "backend developer", | |
| "frontend developer", | |
| "data scientist", | |
| "full stack developer", | |
| "devops engineer", | |
| "ui ux designer", | |
| "mobile developer", | |
| # Non-IT (untuk variasi dataset di luar teknologi) | |
| "marketing", | |
| "accounting", | |
| "customer service", | |
| "sales", | |
| "finance", | |
| ] | |
| def push(jobs, source): | |
| batch = [] | |
| seen = set() | |
| for job in jobs: | |
| job = clean_job(job) | |
| if not job: | |
| continue | |
| url = job["url"] | |
| if url in seen: | |
| continue | |
| seen.add(url) | |
| batch.append(normalize(job, source)) | |
| if not batch: | |
| return 0 | |
| try: | |
| supabase.table("jobs") \ | |
| .upsert(batch, on_conflict="url") \ | |
| .execute() | |
| except Exception as e: | |
| log.error("[Supabase] Upsert gagal (%s): %s", source, e) | |
| return 0 | |
| return len(batch) | |
| def scrape_source(scraper, source, keyword): | |
| try: | |
| jobs = scraper(keyword) | |
| except Exception as e: | |
| log.error("[%s] Scraping gagal (keyword=%s): %s", source, keyword, e) | |
| return | |
| inserted = push(jobs, source) | |
| log.info("[%s] %s: %d ditemukan, %d dimasukkan", source, keyword, len(jobs), inserted) | |
| def main(): | |
| for keyword in KEYWORDS: | |
| scrape_source(scrape_linkedin, "LinkedIn", keyword) | |
| scrape_source(scrape_jobstreet, "JobStreet", keyword) | |
| running = True | |
| def shutdown(signum, frame): | |
| global running | |
| log.info("Ctrl+C ditekan. Menghentikan scraper...") | |
| running = False | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser(description="Job scraper LinkedIn & JobStreet") | |
| parser.add_argument("--once", action="store_true", | |
| help="Jalankan sekali lalu keluar (untuk cron/CI)") | |
| args = parser.parse_args() | |
| signal.signal(signal.SIGINT, shutdown) | |
| signal.signal(signal.SIGTERM, shutdown) | |
| main() # jalankan langsung saat pertama kali start | |
| if args.once: | |
| sys.exit(0) | |
| schedule.every(24).hours.do(main) | |
| log.info("Job scraper berjalan setiap 24 jam...") | |
| log.info("Tekan Ctrl+C untuk menghentikan.") | |
| while running: | |
| schedule.run_pending() | |
| time.sleep(60) | |
| log.info("Scraper dihentikan. Sampai jumpa!") | |
| sys.exit(0) |