File size: 4,720 Bytes
e84ab4f
 
079f46c
c775b3d
e84ab4f
 
 
 
c775b3d
079f46c
e84ab4f
079f46c
 
e84ab4f
 
 
079f46c
c775b3d
e84ab4f
079f46c
e84ab4f
 
079f46c
e84ab4f
 
 
079f46c
e84ab4f
 
 
079f46c
 
e84ab4f
 
 
 
 
c775b3d
e84ab4f
c775b3d
e84ab4f
 
 
c775b3d
 
e84ab4f
 
 
 
c775b3d
e84ab4f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c775b3d
 
e84ab4f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c775b3d
e84ab4f
 
079f46c
e84ab4f
079f46c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
import logging

from bs4 import BeautifulSoup
from urllib.parse import urljoin

from base import goto_with_retry, human_delay, jitter_scroll, short_delay
from browser import get_browser
from extract import extract_contact, extract_job_type, extract_salary
from skills_keyword import extract_skills

log = logging.getLogger(__name__)


def scrape_linkedin(keyword="software engineer", max_jobs=25):
    p, browser, context = get_browser()
    jobs = []

    try:
        page = context.new_page()

        base_url = "https://www.linkedin.com"
        url = f"{base_url}/jobs/search/?keywords={keyword}&location=Indonesia"

        if not goto_with_retry(page, url):
            log.warning("[LinkedIn] Gagal memuat halaman pencarian: %s", url)
            return jobs

        human_delay()
        jitter_scroll(page)
        human_delay()

        try:
            page.wait_for_selector("div.base-card", timeout=30000)
        except Exception:
            log.warning("[LinkedIn] Kartu lowongan tidak muncul untuk keyword: %s", keyword)

        soup = BeautifulSoup(page.content(), "html.parser")

        cards = soup.select("div.base-card")

        for card in cards:
            if len(jobs) >= max_jobs:
                break

            try:
                title_el = card.select_one(
                    "h3.base-search-card__title, "
                    "span.screen-reader-text, "
                    "h3"
                )
                company_el = card.select_one(
                    "h4.base-search-card__subtitle, "
                    "a.hidden-nested-link, "
                    "h4"
                )
                location_el = card.select_one(
                    "span.job-search-card__location, "
                    "span[class*='location']"
                )
                link_el = card.select_one("a.base-card__full-link, a[href*='/jobs/view/']")

                if not (title_el and company_el and link_el):
                    continue

                href = link_el.get("href", "")
                if href.startswith("http"):
                    job_url = href.split("?")[0]
                else:
                    job_url = urljoin(base_url, href.split("?")[0])

                location_text = location_el.text.strip() if location_el else ""

                description = ""
                salary = None
                job_type = "Full-time"
                contact_info = None

                try:
                    detail_page = context.new_page()
                    detail_page.goto(job_url, wait_until="domcontentloaded", timeout=15000)
                    short_delay()
                    detail_soup = BeautifulSoup(detail_page.content(), "html.parser")

                    desc_el = detail_soup.select_one(
                        "div.description__text, "
                        "div[class*='description'], "
                        "section[class*='description']"
                    )
                    if desc_el:
                        description = desc_el.text.strip()

                    page_text = detail_soup.get_text(" ", strip=True)

                    salary = extract_salary(description or page_text)
                    job_type = extract_job_type(page_text)
                    contact_info = extract_contact(description or page_text)

                    if not location_text:
                        loc_el = detail_soup.select_one(
                            "div[class*='location'], "
                            "span[class*='location']"
                        )
                        if loc_el:
                            location_text = loc_el.text.strip()

                    detail_page.close()
                except Exception as e:
                    log.debug("[LinkedIn] Gagal ambil detail %s: %s", job_url, e)

                if not location_text:
                    location_text = "Indonesia"

                jobs.append({
                    "title": title_el.text.strip(),
                    "company": company_el.text.strip(),
                    "location": location_text,
                    "type": job_type,
                    "salary": salary,
                    "url": job_url,
                    "description": description,
                    "skills": extract_skills(description),
                    "contact_info": contact_info,
                })
            except Exception as e:
                log.debug("[LinkedIn] Gagal proses kartu: %s", e)
                continue
    finally:
        try:
            browser.close()
        except Exception:
            pass
        p.stop()

    log.info("[LinkedIn] Keyword '%s' menghasilkan %d lowongan", keyword, len(jobs))
    return jobs