Spaces:
Sleeping
Sleeping
File size: 4,720 Bytes
e84ab4f 079f46c c775b3d e84ab4f c775b3d 079f46c e84ab4f 079f46c e84ab4f 079f46c c775b3d e84ab4f 079f46c e84ab4f 079f46c e84ab4f 079f46c e84ab4f 079f46c e84ab4f c775b3d e84ab4f c775b3d e84ab4f c775b3d e84ab4f c775b3d e84ab4f c775b3d e84ab4f c775b3d e84ab4f 079f46c e84ab4f 079f46c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 | import logging
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from base import goto_with_retry, human_delay, jitter_scroll, short_delay
from browser import get_browser
from extract import extract_contact, extract_job_type, extract_salary
from skills_keyword import extract_skills
log = logging.getLogger(__name__)
def scrape_linkedin(keyword="software engineer", max_jobs=25):
p, browser, context = get_browser()
jobs = []
try:
page = context.new_page()
base_url = "https://www.linkedin.com"
url = f"{base_url}/jobs/search/?keywords={keyword}&location=Indonesia"
if not goto_with_retry(page, url):
log.warning("[LinkedIn] Gagal memuat halaman pencarian: %s", url)
return jobs
human_delay()
jitter_scroll(page)
human_delay()
try:
page.wait_for_selector("div.base-card", timeout=30000)
except Exception:
log.warning("[LinkedIn] Kartu lowongan tidak muncul untuk keyword: %s", keyword)
soup = BeautifulSoup(page.content(), "html.parser")
cards = soup.select("div.base-card")
for card in cards:
if len(jobs) >= max_jobs:
break
try:
title_el = card.select_one(
"h3.base-search-card__title, "
"span.screen-reader-text, "
"h3"
)
company_el = card.select_one(
"h4.base-search-card__subtitle, "
"a.hidden-nested-link, "
"h4"
)
location_el = card.select_one(
"span.job-search-card__location, "
"span[class*='location']"
)
link_el = card.select_one("a.base-card__full-link, a[href*='/jobs/view/']")
if not (title_el and company_el and link_el):
continue
href = link_el.get("href", "")
if href.startswith("http"):
job_url = href.split("?")[0]
else:
job_url = urljoin(base_url, href.split("?")[0])
location_text = location_el.text.strip() if location_el else ""
description = ""
salary = None
job_type = "Full-time"
contact_info = None
try:
detail_page = context.new_page()
detail_page.goto(job_url, wait_until="domcontentloaded", timeout=15000)
short_delay()
detail_soup = BeautifulSoup(detail_page.content(), "html.parser")
desc_el = detail_soup.select_one(
"div.description__text, "
"div[class*='description'], "
"section[class*='description']"
)
if desc_el:
description = desc_el.text.strip()
page_text = detail_soup.get_text(" ", strip=True)
salary = extract_salary(description or page_text)
job_type = extract_job_type(page_text)
contact_info = extract_contact(description or page_text)
if not location_text:
loc_el = detail_soup.select_one(
"div[class*='location'], "
"span[class*='location']"
)
if loc_el:
location_text = loc_el.text.strip()
detail_page.close()
except Exception as e:
log.debug("[LinkedIn] Gagal ambil detail %s: %s", job_url, e)
if not location_text:
location_text = "Indonesia"
jobs.append({
"title": title_el.text.strip(),
"company": company_el.text.strip(),
"location": location_text,
"type": job_type,
"salary": salary,
"url": job_url,
"description": description,
"skills": extract_skills(description),
"contact_info": contact_info,
})
except Exception as e:
log.debug("[LinkedIn] Gagal proses kartu: %s", e)
continue
finally:
try:
browser.close()
except Exception:
pass
p.stop()
log.info("[LinkedIn] Keyword '%s' menghasilkan %d lowongan", keyword, len(jobs))
return jobs |