Sambhavvvvv commited on
Commit
a66040b
·
verified ·
1 Parent(s): 3895656

Delete main.py

Browse files
Files changed (1) hide show
  1. main.py +0 -2350
main.py DELETED
@@ -1,2350 +0,0 @@
1
- import asyncio
2
- import json
3
- import os
4
- import random
5
- import tempfile
6
- import urllib.parse
7
- import time
8
- import logging
9
- from datetime import datetime, timezone
10
- import concurrent.futures
11
- import re
12
- from typing import Optional
13
-
14
- from dotenv import load_dotenv
15
- load_dotenv() # loads .env from project root → SERPER_API_KEY etc.
16
-
17
- from openai import OpenAI
18
-
19
- import httpx
20
- from fastapi import FastAPI, HTTPException, Query, BackgroundTasks
21
- from pydantic import BaseModel
22
- from starlette.responses import StreamingResponse
23
- from fastapi.middleware.cors import CORSMiddleware
24
- from bs4 import BeautifulSoup
25
-
26
- from school_classifier import classify_job, SCHOOLS, PROGRAM_KEYWORDS
27
- from locations import INDIAN_METROS
28
- from database import save_scraped_jobs, get_jobs_by_timeframe, get_jobs_in_timeframe, cleanup_old_jobs, get_binned_jobs, update_company_ratings, get_rated_companies, _connect
29
-
30
- # Primary engine: Selenium with undetected-chromedriver
31
- import undetected_chromedriver as uc
32
- from selenium.webdriver.support.ui import WebDriverWait
33
- from selenium.webdriver.support import expected_conditions as EC
34
- from selenium.webdriver.common.by import By
35
-
36
- # Fallback engine: Playwright with stealth
37
- from playwright.async_api import async_playwright
38
- from playwright_stealth import Stealth
39
-
40
-
41
- from apscheduler.schedulers.asyncio import AsyncIOScheduler
42
- from apscheduler.triggers.interval import IntervalTrigger
43
-
44
- scheduler = AsyncIOScheduler()
45
-
46
- # ── Logging setup ─────────────────────────────────────────────
47
- logging.basicConfig(
48
- level=logging.INFO,
49
- format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
50
- datefmt="%Y-%m-%d %H:%M:%S",
51
- )
52
- logger = logging.getLogger("internscrapper")
53
- logger.setLevel(logging.INFO)
54
-
55
- app = FastAPI(title="LinkedIn Public Job Scraper")
56
-
57
- app.add_middleware(
58
- CORSMiddleware,
59
- allow_origins=["*"],
60
- allow_credentials=False,
61
- allow_methods=["*"],
62
- allow_headers=["*"],
63
- )
64
-
65
- # ── Shared resources ──────────────────────────────────────────
66
- class ScrapeSession:
67
- def __init__(self):
68
- self.history: list[str] = []
69
- self.queues: list[asyncio.Queue] = []
70
- self.is_active = False
71
- self.task: Optional[asyncio.Task] = None
72
-
73
- def cancel(self):
74
- self.is_active = False
75
- if self.task and not self.task.done():
76
- self.task.cancel()
77
-
78
- global_scrape = ScrapeSession()
79
-
80
- executor = concurrent.futures.ThreadPoolExecutor(max_workers=6)
81
- _scrape_lock = asyncio.Lock()
82
-
83
- # ── Auto-scrape state (updated live so the admin UI can poll progress) ────
84
- _auto_scrape_state: dict = {
85
- "is_running": False,
86
- "current_school": None,
87
- "completed": [],
88
- "failed": [],
89
- "started_at": None,
90
- "finished_at": None,
91
- "cancel_requested": False,
92
- }
93
- _auto_scrape_task: Optional[asyncio.Task] = None
94
-
95
- # ── Auto-scrape history (in-memory log of all sweep runs) ─────
96
- _auto_scrape_history: list[dict] = []
97
-
98
- # HuggingFace Spaces sets PORT=7860 in its environment.
99
- # Locally, uvicorn defaults to 8000 unless overridden.
100
- _SELF_PORT = int(os.environ.get("PORT", "8000"))
101
- _SELF_BASE_URL = f"http://localhost:{_SELF_PORT}"
102
-
103
- se_driver = None # Selenium undetected-chromedriver
104
- pw_browser = None # Playwright browser
105
- pw_stealth_ctx = None # Playwright stealth context manager
106
-
107
- # Cookie warm-up cache — browser-derived cookies with a TTL.
108
- _warm_cookies: dict = {}
109
- _warm_cookies_ts: float = 0.0
110
- _COOKIE_TTL_SECONDS = 600 # 10-minute TTL before re-warming
111
-
112
-
113
- # ── Lifecycle ─────────────────────────────────────────────────
114
-
115
- def _get_chrome_major_version() -> Optional[int]:
116
- """Helper to detect the installed Chrome major version on Windows/Linux to prevent driver mismatch."""
117
- try:
118
- import winreg
119
- # Check user-level install / BLBeacon first
120
- try:
121
- with winreg.OpenKey(winreg.HKEY_CURRENT_USER, r"Software\Google\Chrome\BLBeacon") as key:
122
- version, _ = winreg.QueryValueEx(key, "version")
123
- return int(version.split(".")[0])
124
- except Exception:
125
- pass
126
- # Check system-level WOW64 install
127
- try:
128
- with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Wow6432Node\Microsoft\Windows\CurrentVersion\Uninstall\Google Chrome") as key:
129
- version, _ = winreg.QueryValueEx(key, "DisplayVersion")
130
- return int(version.split(".")[0])
131
- except Exception:
132
- pass
133
- # Check system-level 64-bit install
134
- try:
135
- with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\Google Chrome") as key:
136
- version, _ = winreg.QueryValueEx(key, "DisplayVersion")
137
- return int(version.split(".")[0])
138
- except Exception:
139
- pass
140
- except ImportError:
141
- # Linux/macOS
142
- import subprocess
143
- import re
144
- for cmd in ["google-chrome", "google-chrome-stable", "chromium", "chromium-browser"]:
145
- try:
146
- output = subprocess.check_output([cmd, "--version"], stderr=subprocess.DEVNULL)
147
- version_str = output.decode("utf-8").strip()
148
- match = re.search(r"(\d+)\.", version_str)
149
- if match:
150
- return int(match.group(1))
151
- except Exception:
152
- continue
153
- return None
154
-
155
-
156
- # ── User-Agent helpers ────────────────────────────────────────
157
-
158
- def _build_user_agent(chrome_version: Optional[int] = None) -> str:
159
- """Build a realistic Chrome User-Agent string using the installed version."""
160
- v = chrome_version or _get_chrome_major_version() or 131
161
- return (
162
- f"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
163
- f"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{v}.0.0.0 Safari/537.36"
164
- )
165
-
166
-
167
- def _build_ua_pool() -> list[str]:
168
- """Build a pool of realistic User-Agent strings for rotation."""
169
- v = _get_chrome_major_version() or 131
170
- return [
171
- # Windows Chrome (primary — matches our Selenium fingerprint)
172
- f"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{v}.0.0.0 Safari/537.36",
173
- # Windows Chrome (slightly older minor)
174
- f"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{v-1}.0.0.0 Safari/537.36",
175
- # macOS Chrome
176
- f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{v}.0.0.0 Safari/537.36",
177
- # Linux Chrome
178
- f"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{v}.0.0.0 Safari/537.36",
179
- # Windows Edge (Chromium-based, same engine)
180
- f"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{v}.0.0.0 Safari/537.36 Edg/{v}.0.0.0",
181
- ]
182
-
183
-
184
- _UA_POOL: list[str] = [] # populated at startup
185
-
186
-
187
- # ── Response validation helpers ───────────────────────────────
188
-
189
- def _is_empty_stub(html: str) -> bool:
190
- """Detect LinkedIn's known empty/blocked response patterns.
191
-
192
- Returns True if the response is the 26-byte empty stub, a login wall,
193
- a CAPTCHA page, or any other pattern that indicates zero real content.
194
- """
195
- stripped = html.strip()
196
- # 26-byte empty stub: '<!DOCTYPE html> <!----> '
197
- if len(stripped) < 60 and "<!--" in stripped:
198
- return True
199
- # Login/signup redirect pages
200
- lower = stripped[:2000].lower()
201
- if any(sig in lower for sig in [
202
- "login", "sign in", "signup", "sign up",
203
- "authwall", "auth_wall", "checkpoint",
204
- ]):
205
- return True
206
- return False
207
-
208
-
209
- def _create_selenium_driver():
210
- """Create an undetected Chrome instance (runs in thread pool)."""
211
- options = uc.ChromeOptions()
212
- options.add_argument("--headless=new")
213
- options.add_argument("--no-sandbox")
214
- options.add_argument("--disable-dev-shm-usage")
215
- options.add_argument("--disable-gpu")
216
- options.add_argument("--disable-blink-features=AutomationControlled")
217
- options.add_argument("--window-size=1920,1080")
218
- options.add_argument(f"user-agent={_build_user_agent()}")
219
- version_main = _get_chrome_major_version()
220
- if version_main:
221
- driver = uc.Chrome(options=options, version_main=version_main)
222
- else:
223
- driver = uc.Chrome(options=options)
224
- driver.set_page_load_timeout(30)
225
- return driver
226
-
227
- _AUTO_SCRAPE_SCHOOLS = ["socse", "sob", "sodi", "soepp", "solaw", "sofmca", "solas", "soahp"]
228
-
229
- async def _auto_scrape_all_schools():
230
- global _auto_scrape_state, _auto_scrape_history
231
- logger.info("⏰ Auto-scrape: starting school sweep")
232
-
233
- _auto_scrape_state = {
234
- "is_running": True,
235
- "current_school": None,
236
- "completed": [],
237
- "failed": [],
238
- "started_at": time.time(),
239
- "finished_at": None,
240
- "cancel_requested": False,
241
- }
242
-
243
- # ── Create a history entry for this sweep run ──────────────
244
- sweep_entry = {
245
- "sweep_id": len(_auto_scrape_history) + 1,
246
- "is_running": True,
247
- "current_school": None,
248
- "completed": [],
249
- "failed": [],
250
- "started_at": time.time(),
251
- "finished_at": None,
252
- "cancelled": False,
253
- }
254
- _auto_scrape_history.append(sweep_entry)
255
-
256
- all_scraped_companies = set()
257
-
258
- for school in _AUTO_SCRAPE_SCHOOLS:
259
- if _auto_scrape_state.get("cancel_requested"):
260
- logger.info("⏰ Auto-scrape: cancelled by user")
261
- sweep_entry["cancelled"] = True
262
- break
263
-
264
- _auto_scrape_state["current_school"] = school
265
- sweep_entry["current_school"] = school
266
- try:
267
- logger.info(f"⏰ Auto-scrape: triggering [{school}]")
268
-
269
- params = {
270
- "keywords": school,
271
- "freshness": "r86400",
272
- "work_types": ["onsite", "remote", "hybrid"],
273
- "job_types": ["internship"],
274
- }
275
-
276
- async with httpx.AsyncClient(
277
- base_url=_SELF_BASE_URL,
278
- timeout=None,
279
- ) as client:
280
- async with client.stream(
281
- "POST",
282
- "/scrape-internships",
283
- params=params,
284
- ) as resp:
285
- total = 0
286
- async for line in resp.aiter_lines():
287
- if not line.strip():
288
- continue
289
- try:
290
- event = json.loads(line)
291
- etype = event.get("type")
292
- if etype == "info":
293
- logger.info(f"⏰ [{school}] {event.get('message')}")
294
- elif etype == "jobs":
295
- jobs_data = event.get("data", [])
296
- total += len(jobs_data)
297
- for j in jobs_data:
298
- if j.get("company"):
299
- all_scraped_companies.add(j["company"])
300
- logger.info(f"⏰ [{school}] +{len(jobs_data)} jobs streamed")
301
- elif etype == "done":
302
- total = event.get("total", total)
303
- logger.info(
304
- f"⏰ [{school}] done — total={total} "
305
- f"engine={event.get('engine')} "
306
- f"filtered_out={event.get('filtered_out')}"
307
- )
308
- elif etype == "error":
309
- logger.error(f"⏰ [{school}] error: {event.get('message')}")
310
- except Exception:
311
- pass
312
-
313
- _auto_scrape_state["completed"].append({"school": school, "jobs": total})
314
- sweep_entry["completed"].append({"school": school, "jobs": total})
315
- await asyncio.sleep(random.uniform(10.0, 20.0))
316
-
317
- except Exception as e:
318
- logger.error(f"⏰ Auto-scrape [{school}]: failed — {e}")
319
- _auto_scrape_state["failed"].append({"school": school, "error": str(e)})
320
- sweep_entry["failed"].append({"school": school, "error": str(e)})
321
- continue
322
-
323
- _auto_scrape_state["is_running"] = False
324
- _auto_scrape_state["current_school"] = None
325
- _auto_scrape_state["finished_at"] = time.time()
326
-
327
- # ── Finalize the history entry ─────────────────────────────
328
- sweep_entry["is_running"] = False
329
- sweep_entry["current_school"] = None
330
- sweep_entry["finished_at"] = time.time()
331
- logger.info("⏰ Auto-scrape: sweep complete")
332
-
333
- if all_scraped_companies and not _auto_scrape_state.get("cancel_requested"):
334
- try:
335
- loop = asyncio.get_event_loop()
336
- already_rated = await loop.run_in_executor(executor, get_rated_companies)
337
- unrated_companies = [
338
- c for c in all_scraped_companies
339
- if c.lower() not in already_rated
340
- ]
341
- logger.info(
342
- f"⏰ Auto-scrape: {len(all_scraped_companies)} companies scraped, "
343
- f"{len(already_rated)} already rated in DB, "
344
- f"rating {len(unrated_companies)} new ones..."
345
- )
346
- if unrated_companies:
347
- ratings = await process_company_ratings(unrated_companies)
348
- await loop.run_in_executor(executor, _persist_ratings, ratings)
349
- else:
350
- logger.info("⏰ Auto-scrape: all companies already rated — skipping Serper search")
351
- except Exception as e:
352
- logger.error(f"⏰ Auto-scrape: failed to rate companies: {e}")
353
-
354
-
355
- @app.on_event("startup")
356
- async def startup_event():
357
- global se_driver, pw_browser, pw_stealth_ctx, _UA_POOL
358
- loop = asyncio.get_event_loop()
359
-
360
- # 0. Build the UA rotation pool
361
- _UA_POOL = _build_ua_pool()
362
- logger.info(f"✓ UA pool built ({len(_UA_POOL)} variants, Chrome v{_get_chrome_major_version() or '?'})")
363
-
364
- # 1. Primary: Selenium (best anti-detection for LinkedIn)
365
- try:
366
- se_driver = await loop.run_in_executor(executor, _create_selenium_driver)
367
- logger.info("✓ Selenium undetected-chromedriver ready")
368
- except Exception as e:
369
- logger.warning(f"✗ Selenium init failed (Chrome installed?): {e}")
370
-
371
- # 2. Fallback: Playwright + Stealth
372
- try:
373
- stealth = Stealth()
374
- pw_stealth_ctx = stealth.use_async(async_playwright())
375
- pw = await pw_stealth_ctx.__aenter__()
376
- pw_browser = await pw.chromium.launch(
377
- headless=True,
378
- args=["--disable-blink-features=AutomationControlled"],
379
- )
380
- logger.info("✓ Playwright stealth browser ready")
381
- except Exception as e:
382
- logger.warning(f"✗ Playwright init failed: {e}")
383
- if not scheduler.running:
384
- scheduler.start()
385
- logger.info("✓ Scheduler started (awaiting manual trigger)")
386
- else:
387
- logger.info("⚠ Scheduler already running, skipping start")
388
-
389
-
390
-
391
- @app.on_event("shutdown")
392
- async def shutdown_event():
393
- global se_driver, pw_browser, pw_stealth_ctx
394
- scheduler.shutdown(wait=False)
395
- loop = asyncio.get_event_loop()
396
-
397
- if se_driver:
398
- await loop.run_in_executor(executor, se_driver.quit)
399
- if pw_browser:
400
- await pw_browser.close()
401
- if pw_stealth_ctx:
402
- await pw_stealth_ctx.__aexit__(None, None, None)
403
-
404
-
405
- # ── Cookie warm-up ────────────────────────────────────────────
406
-
407
- def _selenium_get_cookies(driver, url: str) -> dict:
408
- """Navigate to a URL and extract cookies (runs in executor thread)."""
409
- driver.get(url)
410
- time.sleep(random.uniform(2.0, 4.0)) # let LinkedIn set session cookies
411
- try:
412
- return {c["name"]: c["value"] for c in driver.get_cookies()}
413
- except Exception:
414
- return {}
415
-
416
-
417
- async def _warm_up_cookies(force: bool = False) -> dict:
418
- """Obtain fresh LinkedIn session cookies via a browser visit.
419
-
420
- Uses Selenium (preferred) or Playwright to visit a simple LinkedIn
421
- guest page, allowing LinkedIn to set its session/tracking cookies
422
- (JSESSIONID, bcookie, li_gc, etc.). These cookies are then forwarded
423
- to the httpx HTTP client for API requests.
424
-
425
- Results are cached for _COOKIE_TTL_SECONDS (10 min) to avoid
426
- redundant browser visits on consecutive scrapes.
427
- """
428
- global _warm_cookies, _warm_cookies_ts, se_driver
429
-
430
- # Return cached cookies if still fresh
431
- if not force and _warm_cookies and (time.time() - _warm_cookies_ts) < _COOKIE_TTL_SECONDS:
432
- logger.info("cookie warm-up: using cached cookies (still fresh)")
433
- return _warm_cookies
434
-
435
- warm_url = "https://www.linkedin.com/jobs/search/?keywords=intern&f_TPR=r86400"
436
- cookies: dict = {}
437
-
438
- # Try Selenium first
439
- if se_driver:
440
- try:
441
- loop = asyncio.get_event_loop()
442
- cookies = await loop.run_in_executor(
443
- executor, _selenium_get_cookies, se_driver, warm_url
444
- )
445
- if cookies:
446
- logger.info(f"cookie warm-up: got {len(cookies)} cookies via Selenium")
447
- except Exception as e:
448
- logger.warning(f"cookie warm-up: Selenium failed: {e}")
449
- err_str = str(e).lower()
450
- if "window" in err_str or "closed" in err_str or "view" in err_str:
451
- logger.info("cookie warm-up: Recreating crashed Selenium driver...")
452
- try:
453
- loop = asyncio.get_event_loop()
454
- try:
455
- await loop.run_in_executor(executor, se_driver.quit)
456
- except Exception:
457
- pass
458
- se_driver = await loop.run_in_executor(executor, _create_selenium_driver)
459
- cookies = await loop.run_in_executor(
460
- executor, _selenium_get_cookies, se_driver, warm_url
461
- )
462
- if cookies:
463
- logger.info(f"cookie warm-up: got {len(cookies)} cookies via recreated Selenium")
464
- except Exception as e2:
465
- logger.warning(f"cookie warm-up: Selenium recreation failed: {e2}")
466
-
467
- # Fall back to Playwright
468
- if not cookies and pw_browser:
469
- try:
470
- ctx = await pw_browser.new_context(user_agent=_build_user_agent())
471
- page = await ctx.new_page()
472
- await page.goto(warm_url, wait_until="domcontentloaded", timeout=20000)
473
- await asyncio.sleep(random.uniform(2.0, 4.0))
474
- cookies = {c["name"]: c["value"] for c in await ctx.cookies()}
475
- await page.close()
476
- await ctx.close()
477
- if cookies:
478
- logger.info(f"cookie warm-up: got {len(cookies)} cookies via Playwright")
479
- except Exception as e:
480
- logger.warning(f"cookie warm-up: Playwright failed: {e}")
481
-
482
- if cookies:
483
- _warm_cookies = cookies
484
- _warm_cookies_ts = time.time()
485
- else:
486
- logger.warning("cookie warm-up: no cookies obtained from any engine")
487
-
488
- return cookies
489
-
490
-
491
- # ── URL builder ───────────────────────────────────────────────
492
- #
493
- # Reference: https://www.linkedin.com/jobs/search/ guest-search parameter spec
494
- # sortBy : R = relevance (fixed)
495
- # f_E : 1 = Internship experience level (fixed)
496
- # geoId : LinkedIn region id — narrows location (fixed, Bengaluru region)
497
- # f_PP : comma-joined Primary-Place ids — UI-only narrowing (fixed)
498
- # f_TPR : r<seconds> freshness — variable (dropdown)
499
- # f_WT : comma-joined work-type codes — variable (3-checkbox group)
500
- # f_JT : comma-joined job-type codes — variable (2-checkbox group)
501
- #
502
- # Per project spec, location is fixed (no city picker) and we only expose
503
- # three user-visible filters: work-type, job-type, freshness.
504
- #
505
- # IMPORTANT — geoId vs f_PP split:
506
- # • The full UI page at /jobs/search/ respects BOTH geoId AND f_PP and uses
507
- # f_PP to narrow within a broader geoId. We keep both on the UI URL so the
508
- # "Open in LinkedIn" link matches the user's reference URL.
509
- # • The guest /jobs-guest/jobs/api/seeMoreJobPostings/search endpoint
510
- # SILENTLY returns an empty 26-byte stub when f_PP is present (it does
511
- # not support the f_PP filter), so the HTTP-pagination path strips f_PP
512
- # and relies on geoId only. See _PAGINATION_PARAM_KEYS below.
513
-
514
- # Always-on params. geoId 90009633 = "Bengaluru" (covers Bengaluru metro on
515
- # the guest API). f_PP narrows further on the UI URL for the "Open in
516
- # LinkedIn" link; the API silently ignores it so we strip it for pagination.
517
- _FIXED_PARAMS: dict[str, str] = {
518
- "sortBy": "R",
519
- "f_E": "1",
520
- "geoId": "90009633",
521
- }
522
-
523
- # Work-type checkbox options (label → LinkedIn f_WT code).
524
- WORK_TYPES: dict[str, str] = {
525
- "onsite": "1",
526
- "remote": "2",
527
- "hybrid": "3",
528
- }
529
-
530
- # Job-type checkbox options (label → LinkedIn f_JT code) — spec restricts to
531
- # these two only.
532
- JOB_TYPES: dict[str, str] = {
533
- "internship": "I",
534
- "full_time": "F",
535
- }
536
-
537
- # Freshness dropdown presets (label → LinkedIn f_TPR code).
538
- FRESHNESS_PRESETS: dict[str, str] = {
539
- "hour": "r3600",
540
- "day": "r86400",
541
- "week": "r604800",
542
- "month": "r2592000",
543
- }
544
-
545
- # ── School-based search term generation ───────────────────────
546
- #
547
- # Convenient shortcut keywords that resolve to a specific school.
548
- # The user specifically requested that searches for a school (e.g. "SOB")
549
- # should search EXACTLY these keywords on LinkedIn.
550
- KEYWORD_TO_SCHOOL: dict[str, str] = {
551
- # [[SOCSE]] — Computer Science & Engineering
552
- "software": "SOCSE",
553
- "cs": "SOCSE",
554
- "computer science": "SOCSE",
555
- "programming": "SOCSE",
556
- "coding": "SOCSE",
557
- "data": "SOCSE",
558
- "data science": "SOCSE",
559
- "machine learning": "SOCSE",
560
- "ai": "SOCSE",
561
- "cyber": "SOCSE",
562
- "cybersecurity": "SOCSE",
563
- "qa": "SOCSE",
564
- "devops": "SOCSE",
565
- "frontend": "SOCSE",
566
- "backend": "SOCSE",
567
- "fullstack": "SOCSE",
568
- "cloud": "SOCSE",
569
- "developer": "SOCSE",
570
- "engineer": "SOCSE",
571
- "sde": "SOCSE",
572
- "swe": "SOCSE",
573
- "it": "SOCSE",
574
- "web development": "SOCSE",
575
- "app development": "SOCSE",
576
- "mobile development": "SOCSE",
577
- "android": "SOCSE",
578
- "ios": "SOCSE",
579
- "react": "SOCSE",
580
- "node": "SOCSE",
581
- "python": "SOCSE",
582
- "java": "SOCSE",
583
- "c++": "SOCSE",
584
- "golang": "SOCSE",
585
- "flutter": "SOCSE",
586
- "aws": "SOCSE",
587
- "azure": "SOCSE",
588
- "gcp": "SOCSE",
589
- "data analyst": "SOCSE",
590
- "data engineer": "SOCSE",
591
- "nlp": "SOCSE",
592
- "computer vision": "SOCSE",
593
- "quality assurance": "SOCSE",
594
- "testing": "SOCSE",
595
- "blockchain": "SOCSE",
596
- "web3": "SOCSE",
597
- "systems engineer": "SOCSE",
598
- "network engineer": "SOCSE",
599
- "site reliability": "SOCSE",
600
- "sre": "SOCSE",
601
- "hardware": "SOCSE",
602
- "embedded": "SOCSE",
603
- # SODI — Design & Innovation
604
- "designing": "SOCSE,SODI",
605
- "ui ux": "SOCSE,SODI",
606
- "graphic designer": "SOCSE,SODI",
607
- "ui designer": "SOCSE,SODI",
608
- "ux designer": "SOCSE,SODI",
609
- "product designer": "SOCSE,SODI",
610
- "product design": "SODI",
611
-
612
- "industrial design":"SODI",
613
- "interaction design":"SODI",
614
- "visual design": "SODI",
615
- # SOB — Business
616
- "business": "SOB",
617
- "marketing": "SOB",
618
- "finance": "SOB",
619
- "hr": "SOB",
620
- "accounting": "SOB",
621
- "commerce": "SOB",
622
- "mba": "SOB",
623
- "sales": "SOB",
624
- "strategy": "SOB",
625
- "operations": "SOB",
626
- "management": "SOB",
627
- "consulting": "SOB",
628
- "analyst": "SOB",
629
- "business analyst": "SOB",
630
- "business development": "SOB",
631
- "bda": "SOB",
632
- "bde": "SOB",
633
- "sales intern": "SOB",
634
- "marketing intern": "SOB",
635
- "hr intern": "SOB",
636
- "finance intern": "SOB",
637
- "investment banking": "SOB",
638
- "venture capital": "SOB",
639
- "private equity": "SOB",
640
- "digital marketing": "SOB",
641
- "seo": "SOB",
642
- "content writer": "SOB",
643
- "social media": "SOB",
644
- "growth": "SOB",
645
- "product manager": "SOB",
646
- "project manager": "SOB",
647
- # SoEPP — Economics & Public Policy
648
- "economics": "SoEPP",
649
- "policy": "SoEPP",
650
- "governance": "SoEPP",
651
- "public policy": "SoEPP",
652
- "research": "SoEPP",
653
- "development studies":"SoEPP",
654
- "economist": "SoEPP",
655
- "policy analyst": "SoEPP",
656
- "public relations": "SoEPP",
657
- "government": "SoEPP",
658
- # SOLaw — Law
659
- "law": "SOLaw",
660
- "legal": "SOLaw",
661
- "paralegal": "SOLaw",
662
- "attorney": "SOLaw",
663
- "counsel": "SOLaw",
664
- "cyber law": "SOLaw",
665
- "law intern": "SOLaw",
666
- "legal intern": "SOLaw",
667
- "litigation": "SOLaw",
668
- "corporate law": "SOLaw",
669
- "ipr": "SOLaw",
670
- # SOFMCA — Film, Media & Creative Arts
671
- "media": "SOFMCA",
672
- "film": "SOFMCA",
673
- "journalism": "SOFMCA",
674
- "animation": "SOFMCA",
675
- "vfx": "SOFMCA",
676
- "gaming": "SOFMCA",
677
- "content creation": "SOFMCA",
678
- "acting": "SOFMCA",
679
- "editor": "SOFMCA",
680
- "video editor": "SOFMCA",
681
- "cinematographer": "SOFMCA",
682
- "copywriter": "SOFMCA",
683
- "media intern": "SOFMCA",
684
- "pr intern": "SOFMCA",
685
- "producer": "SOFMCA",
686
- # SOLAS — Liberal Arts & Sciences
687
- "psychology": "SOLAS",
688
- "environment": "SOLAS",
689
- "liberal arts": "SOLAS",
690
- "sociology": "SOLAS",
691
- "history": "SOLAS",
692
- "behavioral science":"SOLAS",
693
- "psychology intern": "SOLAS",
694
- "counseling": "SOLAS",
695
- "sociologist": "SOLAS",
696
- "research assistant":"SOLAS",
697
- # SOAHP — Allied & Healthcare
698
- "healthcare": "SOAHP",
699
- "medical": "SOAHP",
700
- "clinical": "SOAHP",
701
- "laboratory": "SOAHP",
702
- "nursing": "SOAHP",
703
- "public health": "SOAHP",
704
- "clinical research": "SOAHP",
705
- "public health intern": "SOAHP",
706
- "hospital administration": "SOAHP",
707
- "pharma": "SOAHP",
708
- }
709
-
710
- # Build SCHOOL_SEARCH_TERMS directly from KEYWORD_TO_SCHOOL.
711
- # The user explicitly wants us to search *exactly* these keywords.
712
- SCHOOL_SEARCH_TERMS: dict[str, list[str]] = {}
713
- for kw, school_str in KEYWORD_TO_SCHOOL.items():
714
- for school in school_str.split(","):
715
- school_lower = school.strip().lower()
716
- if school_lower not in SCHOOL_SEARCH_TERMS:
717
- SCHOOL_SEARCH_TERMS[school_lower] = []
718
- SCHOOL_SEARCH_TERMS[school_lower].append(kw)
719
-
720
-
721
- def build_search_params(
722
- keywords: str,
723
- freshness: str = "r86400",
724
- work_types: Optional[list[str]] = None,
725
- job_types: Optional[list[str]] = None,
726
- ) -> dict[str, str]:
727
- """Build the LinkedIn guest search query-param dict.
728
-
729
- Fixed params (sortBy, f_E, f_PP) are always present.
730
- Variable params reflect user-selected checkboxes / dropdown:
731
- - f_TPR : freshness (raw "r<seconds>" code)
732
- - f_WT : comma-joined work-type codes (only when any box ticked)
733
- - f_JT : comma-joined job-type codes (only when any box ticked)
734
- """
735
- params: dict[str, str] = {
736
- "keywords": keywords,
737
- **_FIXED_PARAMS,
738
- "f_TPR": freshness,
739
- }
740
-
741
- if work_types:
742
- codes = [WORK_TYPES[k] for k in work_types if k in WORK_TYPES]
743
- if codes:
744
- params["f_WT"] = ",".join(codes)
745
-
746
- if job_types:
747
- codes = [JOB_TYPES[k] for k in job_types if k in JOB_TYPES]
748
- if codes:
749
- params["f_JT"] = ",".join(codes)
750
-
751
- return params
752
-
753
-
754
- def build_search_url(
755
- keywords: str,
756
- freshness: str = "r86400",
757
- work_types: Optional[list[str]] = None,
758
- job_types: Optional[list[str]] = None,
759
- ) -> str:
760
- """Builds a fully-parameterised LinkedIn guest search URL.
761
-
762
- Shape matches LinkedIn's UI-canonical form, e.g.:
763
- https://www.linkedin.com/jobs/search/?keywords=intern&sortBy=R&f_E=1
764
- &f_PP=105214831,113968072,112565523,119634689&f_TPR=r86400&f_WT=1,2&f_JT=I
765
- """
766
- params = build_search_params(
767
- keywords=keywords,
768
- freshness=freshness,
769
- work_types=work_types,
770
- job_types=job_types,
771
- )
772
- # Trailing slash on /jobs/search/ matches LinkedIn's UI-canonical URL.
773
- return f"https://www.linkedin.com/jobs/search/?{urllib.parse.urlencode(params)}"
774
-
775
-
776
- # ── Shared HTML parser ────────────────────────────────────────
777
-
778
- def _parse_jobs_from_html(html: str) -> list[dict]:
779
- """Extract job listings from raw HTML using BeautifulSoup.
780
- Cascading selectors handle multiple LinkedIn page variants.
781
-
782
- NOTE: contact_details (hiring team links) are NOT extracted here.
783
- They live on each job's *detail* page, not the search results list.
784
- Use _fetch_hiring_links_for_jobs() after collection to enrich jobs.
785
- """
786
- soup = BeautifulSoup(html, "html.parser")
787
- jobs = []
788
-
789
- # Primary selector: public guest search results
790
- cards = soup.select("ul.jobs-search__results-list > li")
791
-
792
- # Fallback selectors for alternate page states + seeMoreJobPostings fragments
793
- # (the paginated endpoint returns bare <li> elements with no wrapping <ul>).
794
- if not cards:
795
- cards = soup.select("li.job-search-card, li.result-card")
796
- if not cards:
797
- cards = soup.select("div.base-card.base-search-card")
798
- if not cards:
799
- cards = soup.select("[data-entity-urn*='jobPosting']")
800
-
801
- for card in cards:
802
- try:
803
- # Title
804
- title_el = (
805
- card.select_one(".base-search-card__title")
806
- or card.select_one("h3")
807
- or card.select_one("[class*='title']")
808
- )
809
- # Company
810
- company_el = (
811
- card.select_one(".base-search-card__subtitle a")
812
- or card.select_one(".base-search-card__subtitle")
813
- or card.select_one("h4 a")
814
- )
815
- # Location
816
- loc_el = (
817
- card.select_one(".job-search-card__location")
818
- or card.select_one("[class*='location']")
819
- )
820
- # Link
821
- link_el = (
822
- card.select_one("a.base-card__full-link")
823
- or card.select_one("a[href*='/jobs/view/']")
824
- or card.select_one("a[href*='linkedin.com/jobs']")
825
- )
826
- # Posted date
827
- date_el = (
828
- card.select_one("time.job-search-card__listdate")
829
- or card.select_one("time.job-search-card__listdate--new")
830
- or card.select_one("time")
831
- )
832
-
833
- title = title_el.get_text(strip=True) if title_el else None
834
- company = company_el.get_text(strip=True) if company_el else None
835
- location = loc_el.get_text(strip=True) if loc_el else None
836
- raw_link = link_el.get("href", "") if link_el else ""
837
- link = raw_link.split("?")[0] if raw_link else None
838
- posted = date_el.get_text(strip=True) if date_el else None
839
- posted_dt = date_el.get("datetime") if date_el else None
840
-
841
- # At least title or company must exist for a valid card
842
- if title or company:
843
- classification = classify_job(title or "", company or "")
844
- jobs.append({
845
- "title": title,
846
- "company": company,
847
- "location": location,
848
- "link": link,
849
- "posted": posted,
850
- "posted_datetime": posted_dt,
851
- "programs": classification["programs"],
852
- "schools": classification["schools"],
853
- "contact_details": [], # enriched later by _fetch_hiring_links_for_jobs
854
- })
855
- except Exception:
856
- continue
857
-
858
- return jobs
859
-
860
-
861
-
862
- # ── Serper-based hiring manager search ───────────────────────
863
-
864
- _SERPER_API_KEY = os.environ.get("SERPER_API_KEY", "0820e51e9080c289b3849e30189e023774fdad87")
865
- _SERPER_URL = "https://google.serper.dev/search"
866
-
867
- # Semaphore: at most 3 concurrent Serper requests
868
- _SERPER_SEM = asyncio.Semaphore(3)
869
-
870
- _LINKEDIN_IN_RE = re.compile(r'https?://(?:[\w-]+\.)?linkedin\.com/in/([\w%-]+)', re.IGNORECASE)
871
-
872
-
873
- def _extract_linkedin_profile_from_serper(data: dict) -> dict | None:
874
- """Parse a Serper JSON response and return the first LinkedIn /in/ profile found.
875
-
876
- Priority order:
877
- 1. answerBox.link (Google AI / Featured Snippet)
878
- 2. answerBox.snippet (text contains a linkedin.com/in/ URL)
879
- 3. organic[0].link (first organic result)
880
- 4. organic[].link (first organic result that is a /in/ URL)
881
-
882
- Returns:
883
- {"name": "<person_slug_or_extracted_name>", "url": "<linkedin_profile_url>"}
884
- or None if nothing found.
885
- """
886
- # 1. answerBox direct link
887
- ab = data.get("answerBox", {})
888
- ab_link = ab.get("link", "")
889
- if ab_link and "linkedin.com/in/" in ab_link:
890
- slug = ab_link.rstrip("/").split("/in/")[-1].split("?")[0]
891
- name = ab.get("title") or slug.replace("-", " ").title()
892
- return {"name": name, "url": ab_link.split("?")[0]}
893
-
894
- # 2. answerBox snippet contains a URL
895
- ab_snippet = ab.get("snippet", "") or ab.get("answer", "")
896
- m = _LINKEDIN_IN_RE.search(ab_snippet)
897
- if m:
898
- url = m.group(0).split("?")[0]
899
- slug = m.group(1)
900
- name = ab.get("title") or slug.replace("-", " ").title()
901
- return {"name": name, "url": url}
902
-
903
- # 3 & 4. Organic results — prefer linkedin.com/in/ links
904
- for result in data.get("organic", []):
905
- link = result.get("link", "")
906
- if "linkedin.com/in/" in link:
907
- slug = link.rstrip("/").split("/in/")[-1].split("?")[0]
908
- name = result.get("title", slug).split(" - ")[0].split(" | ")[0].strip()
909
- return {"name": name, "url": link.split("?")[0]}
910
-
911
- return None
912
-
913
-
914
- async def _fetch_hiring_contacts_via_serper(jobs: list[dict]) -> None:
915
- """Enrich each job dict in-place with contact_details via Serper Google search.
916
-
917
- For each job, searches:
918
- site:linkedin.com/in "{company}" "hiring" OR "recruiter" OR "HR"
919
-
920
- contact_details becomes a list with at most 1 entry:
921
- [{"name": "Person Name", "url": "https://linkedin.com/in/slug"}]
922
-
923
- Jobs where no profile is found keep contact_details as [].
924
- Requires SERPER_API_KEY in environment.
925
- """
926
- if not _SERPER_API_KEY:
927
- logger.warning("serper: SERPER_API_KEY not set — skipping hiring contact search")
928
- return
929
-
930
- async def _search_one(job: dict) -> None:
931
- company = job.get("company") or ""
932
- if not company:
933
- return
934
-
935
- # Build a targeted query: site operator + company + hiring signals
936
- query = f'site:linkedin.com/in "{company}" hiring OR recruiter OR "HR"'
937
-
938
- try:
939
- async with _SERPER_SEM:
940
- await asyncio.sleep(random.uniform(0.2, 0.6))
941
- async with httpx.AsyncClient(timeout=10.0) as client:
942
- resp = await client.post(
943
- _SERPER_URL,
944
- json={"q": query, "num": 5, "gl": "in", "hl": "en"},
945
- headers={
946
- "X-API-KEY": _SERPER_API_KEY,
947
- "Content-Type": "application/json",
948
- },
949
- )
950
-
951
- if resp.status_code != 200:
952
- logger.debug(
953
- f"serper: HTTP {resp.status_code} for company={company!r}"
954
- )
955
- return
956
-
957
- data = resp.json()
958
- profile = _extract_linkedin_profile_from_serper(data)
959
- if profile:
960
- job["contact_details"] = [profile]
961
- logger.info(
962
- f"serper: ✓ {job.get('title','?')} @ {company} → "
963
- f"{profile['name']} ({profile['url']})"
964
- )
965
- else:
966
- logger.debug(f"serper: no profile found for company={company!r}")
967
-
968
- except Exception as e:
969
- logger.debug(f"serper: error searching for {company!r}: {e}")
970
-
971
- await asyncio.gather(*[_search_one(j) for j in jobs])
972
-
973
-
974
-
975
-
976
- # ── Paginated HTTP fetcher (seeMoreJobPostings) ───────────────
977
-
978
- _SEE_MORE_URL = (
979
- "https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search"
980
- )
981
- # Keys forwarded to the seeMoreJobPostings endpoint.
982
- # NOTE: f_PP is intentionally EXCLUDED here — the guest API returns an empty
983
- # 26-byte stub whenever f_PP is present (verified 2026-05-20). geoId provides
984
- # equivalent narrowing. f_PP stays on the UI URL only.
985
- _PAGINATION_PARAM_KEYS = {
986
- "keywords", "sortBy", "f_E", "geoId", "f_TPR", "f_WT", "f_JT",
987
- }
988
-
989
-
990
- async def _fetch_more_pages(
991
- base_params: dict,
992
- cookies: dict,
993
- headers: dict,
994
- max_pages: int = 20,
995
- start_offset: int = 25,
996
- ) -> list[dict]:
997
- """Page through LinkedIn's guest seeMoreJobPostings endpoint.
998
-
999
- The endpoint returns ~25 job cards as a raw HTML fragment per call.
1000
- Stops on: 4xx/5xx (rate limit, with retry on 429), empty body,
1001
- empty stub detection, or empty parsed result. Returns whatever was
1002
- collected before stopping — never raises.
1003
-
1004
- Retries on transient network errors (DNS, connection timeouts) with
1005
- exponential backoff to handle temporary infrastructure issues.
1006
- """
1007
- params_clean = {
1008
- k: v for k, v in base_params.items()
1009
- if k in _PAGINATION_PARAM_KEYS and v not in (None, "")
1010
- }
1011
-
1012
- collected: list[dict] = []
1013
- rate_limit_retries_left = 2 # total 429s tolerated across the whole loop
1014
- first_response_dumped = False
1015
- dump_path = os.path.join(tempfile.gettempdir(), "linkedin_pagination_first_response.html")
1016
- try:
1017
- async with httpx.AsyncClient(
1018
- timeout=20.0,
1019
- cookies=cookies,
1020
- headers=headers,
1021
- follow_redirects=True,
1022
- http2=False,
1023
- ) as client:
1024
- # Brief settle before first request — reduced from 3s for speed.
1025
- await asyncio.sleep(random.uniform(1.0, 2.0))
1026
-
1027
- start = start_offset
1028
- for page_idx in range(max_pages):
1029
- url = f"{_SEE_MORE_URL}?{urllib.parse.urlencode({**params_clean, 'start': start})}"
1030
- logger.info(f"pagination: GET {url}")
1031
-
1032
- # Retry loop for transient network errors
1033
- max_network_retries = 3
1034
- for attempt in range(1, max_network_retries + 1):
1035
- try:
1036
- resp = await client.get(url)
1037
- break # Success, exit retry loop
1038
- except (httpx.TimeoutException, httpx.NetworkError, OSError) as e:
1039
- # Transient network errors: DNS (getaddrinfo), timeout, connection reset
1040
- if attempt < max_network_retries:
1041
- backoff = min(2 ** attempt, 10.0) * random.uniform(1.0, 1.5)
1042
- logger.warning(
1043
- f"pagination: transient network error at start={start} "
1044
- f"(attempt {attempt}/{max_network_retries}): {type(e).__name__}. "
1045
- f"Retrying in {backoff:.1f}s..."
1046
- )
1047
- await asyncio.sleep(backoff)
1048
- else:
1049
- logger.warning(
1050
- f"pagination: network error at start={start} after "
1051
- f"{max_network_retries} attempts: {e}; stopping with {len(collected)} jobs"
1052
- )
1053
- return collected
1054
- except httpx.HTTPError as e:
1055
- logger.warning(
1056
- f"pagination: HTTP error at start={start}: {e}; "
1057
- f"stopping with {len(collected)} extra jobs"
1058
- )
1059
- return collected
1060
-
1061
- # Retry on 429 with longer backoff — LinkedIn's rate limits
1062
- # are usually short-lived and clear after ~10s.
1063
- if resp.status_code == 429 and rate_limit_retries_left > 0:
1064
- rate_limit_retries_left -= 1
1065
- backoff = random.uniform(15.0, 25.0)
1066
- logger.warning(
1067
- f"pagination: 429 at start={start}; backing off {backoff:.1f}s "
1068
- f"and retrying ({rate_limit_retries_left} retries left)"
1069
- )
1070
- await asyncio.sleep(backoff)
1071
- try:
1072
- resp = await client.get(url)
1073
- except (httpx.TimeoutException, httpx.NetworkError, OSError, httpx.HTTPError) as e:
1074
- logger.warning(f"pagination: retry HTTP error: {e}; stopping")
1075
- return collected
1076
-
1077
- html = resp.text or ""
1078
- body_len = len(resp.content) if resp.content is not None else 0
1079
- li_count = html.lower().count("<li")
1080
-
1081
- logger.info(
1082
- f"pagination: start={start} status={resp.status_code} "
1083
- f"bytes={body_len} <li>={li_count}"
1084
- )
1085
-
1086
- if not first_response_dumped:
1087
- try:
1088
- with open(dump_path, "w", encoding="utf-8") as f:
1089
- f.write(html)
1090
- logger.info(f"pagination: dumped first response → {dump_path}")
1091
- except Exception as e:
1092
- logger.warning(f"pagination: could not dump response: {e}")
1093
- first_response_dumped = True
1094
-
1095
- if resp.status_code >= 400:
1096
- logger.warning(
1097
- f"pagination: status {resp.status_code} at start={start} "
1098
- f"(likely rate-limit); stopping with {len(collected)} extra jobs"
1099
- )
1100
- break
1101
-
1102
- if not html.strip():
1103
- logger.info(
1104
- f"pagination: empty body at start={start}; end of results "
1105
- f"(+{len(collected)} extra jobs)"
1106
- )
1107
- break
1108
-
1109
- # ── Early stub detection ───────────────────────��──
1110
- # Detect the 26-byte empty stub and login walls before
1111
- # wasting time on HTML parsing. On the very first page
1112
- # this signals "HTTP path is blocked" so we can bail fast.
1113
- if _is_empty_stub(html):
1114
- logger.warning(
1115
- f"pagination: empty stub/block detected at start={start} "
1116
- f"(bytes={body_len}); stopping with {len(collected)} extra jobs"
1117
- )
1118
- break
1119
-
1120
- page_jobs = _parse_jobs_from_html(html)
1121
- logger.info(
1122
- f"pagination: start={start} parser returned {len(page_jobs)} jobs"
1123
- )
1124
-
1125
- if not page_jobs:
1126
- snippet = html[:500].replace("\n", " ")
1127
- logger.warning(
1128
- f"pagination: parser returned 0 jobs from a non-empty body "
1129
- f"(bytes={body_len}, <li>={li_count}). First 500 chars: {snippet!r}"
1130
- )
1131
- logger.info(
1132
- f"pagination: no jobs parsed at start={start}; end of results "
1133
- f"(+{len(collected)} extra jobs)"
1134
- )
1135
- break
1136
-
1137
- collected.extend(page_jobs)
1138
- logger.info(
1139
- f"pagination: start={start} → +{len(page_jobs)} jobs "
1140
- f"(running paginated total {len(collected)})"
1141
- )
1142
-
1143
- # Advance by the actual page size LinkedIn returned.
1144
- start += len(page_jobs)
1145
-
1146
- # Human-like inter-page jitter — varied enough to avoid
1147
- # pattern detection.
1148
- await asyncio.sleep(random.uniform(3.5, 6.0))
1149
- except Exception as e:
1150
- logger.warning(
1151
- f"pagination: unexpected error: {e}; returning {len(collected)} extra jobs"
1152
- )
1153
-
1154
- return collected
1155
-
1156
-
1157
- # ── Scraping engines ──────────────────────────────────────────
1158
-
1159
- _JOB_CARD_CSS = (
1160
- "ul.jobs-search__results-list > li, "
1161
- "div.base-card.base-search-card, "
1162
- "[data-entity-urn*='jobPosting']"
1163
- )
1164
-
1165
-
1166
- def _selenium_scrape(driver, url: str) -> tuple[list[dict], dict]:
1167
- """Synchronous Selenium scrape — executed inside executor thread.
1168
- Returns (jobs, cookies) so the async caller can paginate over HTTP."""
1169
- driver.get(url)
1170
-
1171
- # Wait up to 15 s for at least one job card — if nothing loads, bail so
1172
- # Playwright fallback can try instead.
1173
- try:
1174
- WebDriverWait(driver, 15).until(
1175
- EC.presence_of_element_located((By.CSS_SELECTOR, _JOB_CARD_CSS))
1176
- )
1177
- except Exception:
1178
- logger.warning("Selenium: no job cards appeared within 15 s")
1179
- return [], {}
1180
-
1181
- # Scroll to bottom repeatedly; break after 3 consecutive unchanged heights.
1182
- no_change = 0
1183
- for _ in range(15):
1184
- prev_height = driver.execute_script("return document.body.scrollHeight")
1185
- driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
1186
- time.sleep(1.5)
1187
-
1188
- # Click "See more jobs" / "Show more results" if present
1189
- try:
1190
- btn = driver.find_element(
1191
- By.XPATH,
1192
- "//button[contains(translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), 'see more')"
1193
- " or contains(translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), 'show more')]"
1194
- )
1195
- btn.click()
1196
- time.sleep(1.0)
1197
- except Exception:
1198
- pass
1199
-
1200
- new_height = driver.execute_script("return document.body.scrollHeight")
1201
- if new_height == prev_height:
1202
- no_change += 1
1203
- if no_change >= 3:
1204
- break
1205
- else:
1206
- no_change = 0
1207
-
1208
- jobs = _parse_jobs_from_html(driver.page_source)
1209
- try:
1210
- cookies = {c["name"]: c["value"] for c in driver.get_cookies()}
1211
- except Exception as e:
1212
- logger.warning(f"Selenium: could not extract cookies for pagination: {e}")
1213
- cookies = {}
1214
- return jobs, cookies
1215
-
1216
-
1217
- async def _playwright_scrape(browser, url: str) -> tuple[list[dict], dict]:
1218
- """Async Playwright scrape — fallback engine.
1219
- Returns (jobs, cookies) so the caller can paginate over HTTP."""
1220
- context = await browser.new_context(
1221
- user_agent=_build_user_agent()
1222
- )
1223
- page = await context.new_page()
1224
-
1225
- try:
1226
- await page.goto(url, wait_until="domcontentloaded", timeout=30000)
1227
-
1228
- # Wait up to 15 s for at least one job card
1229
- try:
1230
- await page.wait_for_selector(_JOB_CARD_CSS, timeout=15000)
1231
- except Exception:
1232
- logger.warning("Playwright: no job cards appeared within 15 s")
1233
- return [], {}
1234
-
1235
- # Scroll to bottom repeatedly; break after 3 consecutive unchanged heights.
1236
- no_change = 0
1237
- for _ in range(15):
1238
- prev_height = await page.evaluate("document.body.scrollHeight")
1239
- await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
1240
- await asyncio.sleep(1.5)
1241
-
1242
- # Click "See more jobs" button if present
1243
- try:
1244
- await page.locator("button:has-text('See more')").click(timeout=1000)
1245
- await asyncio.sleep(1.0)
1246
- except Exception:
1247
- pass
1248
-
1249
- new_height = await page.evaluate("document.body.scrollHeight")
1250
- if new_height == prev_height:
1251
- no_change += 1
1252
- if no_change >= 3:
1253
- break
1254
- else:
1255
- no_change = 0
1256
-
1257
- html = await page.content()
1258
- jobs = _parse_jobs_from_html(html)
1259
- try:
1260
- cookies = {c["name"]: c["value"] for c in await context.cookies()}
1261
- except Exception as e:
1262
- logger.warning(f"Playwright: could not extract cookies for pagination: {e}")
1263
- cookies = {}
1264
- return jobs, cookies
1265
- finally:
1266
- await page.close()
1267
- await context.close()
1268
-
1269
-
1270
- # ── Single-URL scrape helper (Selenium → Playwright) ─────────
1271
-
1272
- def _build_pagination_headers(referer_url: str) -> dict:
1273
- """Browser-realistic headers for the seeMoreJobPostings endpoint.
1274
- Uses a random UA from the rotation pool and matches the installed
1275
- Chrome version for Sec-Ch-Ua consistency."""
1276
- v = _get_chrome_major_version() or 131
1277
- ua = random.choice(_UA_POOL) if _UA_POOL else _build_user_agent()
1278
- return {
1279
- "User-Agent": ua,
1280
- "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
1281
- "Accept-Language": "en-US,en;q=0.9",
1282
- "Accept-Encoding": "gzip, deflate, br",
1283
- "Referer": referer_url,
1284
- "Sec-Fetch-Dest": "empty",
1285
- "Sec-Fetch-Mode": "cors",
1286
- "Sec-Fetch-Site": "same-origin",
1287
- "Sec-Ch-Ua": f'"Chromium";v="{v}", "Not_A Brand";v="24"',
1288
- "Sec-Ch-Ua-Mobile": "?0",
1289
- "Sec-Ch-Ua-Platform": '"Windows"',
1290
- "Connection": "keep-alive",
1291
- }
1292
-
1293
-
1294
- async def _scrape_one_url(
1295
- url: str,
1296
- params: Optional[dict] = None,
1297
- paginate: bool = False,
1298
- ) -> tuple[list[dict], str]:
1299
- """Scrape one URL with cookie warm-up, exponential-backoff retry,
1300
- and smarter browser fallback.
1301
-
1302
- Pipeline:
1303
- 1. Warm up cookies via browser visit (cached for 10 min).
1304
- 2. Try HTTP pagination with warm cookies (up to 3 attempts
1305
- with exponential backoff on failure).
1306
- 3. If HTTP fails after all retries, fall back to Selenium
1307
- then Playwright to scrape the UI page directly.
1308
- 4. If a browser engine succeeds, try one more HTTP pagination
1309
- pass using the browser's fresh cookies to extend the results.
1310
-
1311
- Returns (jobs, engine_used) where engine_used is one of
1312
- 'http' / 'selenium' / 'playwright' / 'none'.
1313
- """
1314
- global se_driver
1315
- jobs: list[dict] = []
1316
- engine_used = "none"
1317
- browser_cookies: dict = {}
1318
-
1319
- # ── Phase 1 — HTTP with warm cookies + retry loop ─────────────
1320
- if paginate and params:
1321
- max_http_attempts = 3
1322
- for attempt in range(1, max_http_attempts + 1):
1323
- # Warm up cookies (uses cache if fresh; force re-warm on retries)
1324
- cookies = await _warm_up_cookies(force=(attempt > 1))
1325
- logger.info(
1326
- f"HTTP attempt {attempt}/{max_http_attempts} "
1327
- f"(cookies={'warm' if cookies else 'cold'})"
1328
- )
1329
-
1330
- http_jobs = await _fetch_more_pages(
1331
- base_params=params,
1332
- cookies=cookies,
1333
- headers=_build_pagination_headers(referer_url=url),
1334
- max_pages=40,
1335
- start_offset=0,
1336
- )
1337
-
1338
- if http_jobs:
1339
- jobs = http_jobs
1340
- engine_used = "http"
1341
- logger.info(
1342
- f"HTTP scraped {len(jobs)} jobs on attempt {attempt}"
1343
- )
1344
- break
1345
-
1346
- # Exponential backoff before next attempt
1347
- if attempt < max_http_attempts:
1348
- backoff = (2 ** attempt) * random.uniform(2.0, 4.0)
1349
- logger.warning(
1350
- f"HTTP attempt {attempt} returned 0 jobs; "
1351
- f"backing off {backoff:.1f}s before retry"
1352
- )
1353
- await asyncio.sleep(backoff)
1354
-
1355
- if not jobs:
1356
- logger.warning(
1357
- f"HTTP pagination failed after {max_http_attempts} attempts "
1358
- "— falling back to browser engines"
1359
- )
1360
-
1361
- # ── Phase 2 — Browser fallback (Selenium → Playwright) ────────
1362
- if not jobs and se_driver:
1363
- try:
1364
- async with _scrape_lock:
1365
- loop = asyncio.get_event_loop()
1366
- jobs, browser_cookies = await loop.run_in_executor(
1367
- executor, _selenium_scrape, se_driver, url
1368
- )
1369
- if jobs:
1370
- engine_used = "selenium"
1371
- logger.info(f"Selenium scraped {len(jobs)} jobs (fallback)")
1372
- except Exception as e:
1373
- logger.warning(f"Selenium fallback failed: {e}")
1374
- err_str = str(e).lower()
1375
- if "window" in err_str or "closed" in err_str or "view" in err_str:
1376
- logger.info("Selenium fallback: Recreating crashed driver...")
1377
- try:
1378
- async with _scrape_lock:
1379
- loop = asyncio.get_event_loop()
1380
- try:
1381
- await loop.run_in_executor(executor, se_driver.quit)
1382
- except Exception:
1383
- pass
1384
- se_driver = await loop.run_in_executor(executor, _create_selenium_driver)
1385
- jobs, browser_cookies = await loop.run_in_executor(
1386
- executor, _selenium_scrape, se_driver, url
1387
- )
1388
- if jobs:
1389
- engine_used = "selenium"
1390
- logger.info(f"Selenium scraped {len(jobs)} jobs via recreated driver")
1391
- except Exception as e2:
1392
- logger.warning(f"Selenium fallback recreation failed: {e2}")
1393
-
1394
- if not jobs and pw_browser:
1395
- try:
1396
- jobs, browser_cookies = await _playwright_scrape(pw_browser, url)
1397
- if jobs:
1398
- engine_used = "playwright"
1399
- logger.info(f"Playwright scraped {len(jobs)} jobs (fallback)")
1400
- except Exception as e:
1401
- logger.warning(f"Playwright fallback failed: {e}")
1402
-
1403
- # ── Phase 3 — Post-browser HTTP extension ─────────────────────
1404
- # If a browser engine got results, try HTTP pagination with its
1405
- # fresh cookies to collect additional pages beyond what the browser
1406
- # initially loaded (the browser typically only sees page 1).
1407
- if jobs and browser_cookies and paginate and params:
1408
- logger.info(
1409
- f"Extending {len(jobs)} browser results via HTTP with "
1410
- f"{len(browser_cookies)} fresh browser cookies"
1411
- )
1412
- extra_jobs = await _fetch_more_pages(
1413
- base_params=params,
1414
- cookies=browser_cookies,
1415
- headers=_build_pagination_headers(referer_url=url),
1416
- max_pages=20,
1417
- start_offset=len(jobs), # continue from where the browser left off
1418
- )
1419
- if extra_jobs:
1420
- jobs.extend(extra_jobs)
1421
- logger.info(
1422
- f"HTTP extension added {len(extra_jobs)} jobs "
1423
- f"(total now {len(jobs)})"
1424
- )
1425
-
1426
- return jobs, engine_used
1427
-
1428
-
1429
- # ── API endpoints ─────────────────────────────────────────────
1430
-
1431
- @app.get("/schools")
1432
- async def get_schools():
1433
- """Return the full school registry used for job classification."""
1434
- return SCHOOLS
1435
-
1436
-
1437
- @app.get("/locations")
1438
- async def get_locations():
1439
- """Return the city → geoId mapping used for location checkboxes."""
1440
- return INDIAN_METROS
1441
-
1442
-
1443
- # ── Parallel multi-keyword HTTP scraper ───────────────────────
1444
-
1445
- # Concurrency limiter — at most 1 parallel HTTP scrape to avoid
1446
- # triggering LinkedIn's rate limiter. Sequential is safer and still fast.
1447
- _PARALLEL_SEMAPHORE = asyncio.Semaphore(1)
1448
-
1449
-
1450
- async def _fetch_all_pages_for_keyword(
1451
- keyword: str,
1452
- base_params: dict,
1453
- cookies: dict,
1454
- max_pages: int = 10,
1455
- ) -> list[dict]:
1456
- """Fast HTTP-only scraper for a single keyword.
1457
-
1458
- Designed for parallel fan-out: lightweight, no browser, shorter
1459
- inter-page delays. Uses the shared semaphore to throttle concurrency.
1460
-
1461
- Retries on transient network errors with exponential backoff to handle
1462
- temporary infrastructure issues.
1463
- """
1464
- params = {
1465
- k: v for k, v in base_params.items()
1466
- if k in _PAGINATION_PARAM_KEYS and v not in (None, "")
1467
- }
1468
- params["keywords"] = keyword # override with this specific keyword
1469
-
1470
- referer_url = f"https://www.linkedin.com/jobs/search/?{urllib.parse.urlencode(params)}"
1471
- headers = _build_pagination_headers(referer_url=referer_url)
1472
-
1473
- collected: list[dict] = []
1474
-
1475
- async with _PARALLEL_SEMAPHORE:
1476
- try:
1477
- async with httpx.AsyncClient(
1478
- timeout=20.0,
1479
- cookies=cookies,
1480
- headers=headers,
1481
- follow_redirects=True,
1482
- http2=False,
1483
- ) as client:
1484
- # Brief initial settle
1485
- await asyncio.sleep(random.uniform(2.0, 4.0))
1486
-
1487
- start = 0
1488
- consecutive_empty = 0
1489
- for page_idx in range(max_pages):
1490
- url = f"{_SEE_MORE_URL}?{urllib.parse.urlencode({**params, 'start': start})}"
1491
- logger.info(f"parallel[{keyword[:30]}]: GET start={start}")
1492
-
1493
- # Retry loop for transient network errors
1494
- max_network_retries = 3
1495
- resp = None
1496
- for attempt in range(1, max_network_retries + 1):
1497
- try:
1498
- resp = await client.get(url)
1499
- break # Success, exit retry loop
1500
- except (httpx.TimeoutException, httpx.NetworkError, OSError) as e:
1501
- # Transient network errors
1502
- if attempt < max_network_retries:
1503
- backoff = min(2 ** attempt, 10.0) * random.uniform(1.0, 1.5)
1504
- logger.warning(
1505
- f"parallel[{keyword[:30]}]: transient network error "
1506
- f"(attempt {attempt}/{max_network_retries}): {type(e).__name__}. "
1507
- f"Retrying in {backoff:.1f}s..."
1508
- )
1509
- await asyncio.sleep(backoff)
1510
- else:
1511
- logger.warning(
1512
- f"parallel[{keyword[:30]}]: network error at start={start} "
1513
- f"after {max_network_retries} attempts: {e}; "
1514
- f"stopping with {len(collected)} jobs"
1515
- )
1516
- break
1517
- except httpx.HTTPError as e:
1518
- logger.warning(
1519
- f"parallel[{keyword[:30]}]: HTTP error at start={start}: {e}; stopping"
1520
- )
1521
- break
1522
-
1523
- if resp is None:
1524
- break # All retries failed, stop pagination
1525
-
1526
- # Handle rate limits
1527
- if resp.status_code == 429:
1528
- backoff = random.uniform(15.0, 25.0)
1529
- logger.warning(
1530
- f"parallel[{keyword[:30]}]: 429 at start={start}; "
1531
- f"backing off {backoff:.1f}s"
1532
- )
1533
- await asyncio.sleep(backoff)
1534
- try:
1535
- resp = await client.get(url)
1536
- except (httpx.TimeoutException, httpx.NetworkError, OSError, httpx.HTTPError):
1537
- break
1538
-
1539
- if resp.status_code >= 400:
1540
- logger.warning(
1541
- f"parallel[{keyword[:30]}]: status {resp.status_code}; stopping"
1542
- )
1543
- break
1544
-
1545
- html = resp.text or ""
1546
- if not html.strip() or _is_empty_stub(html):
1547
- consecutive_empty += 1
1548
- if consecutive_empty >= 2:
1549
- break
1550
- # One empty might be transient — try next offset
1551
- start += 25
1552
- await asyncio.sleep(random.uniform(0.5, 1.0))
1553
- continue
1554
-
1555
- consecutive_empty = 0
1556
- page_jobs = _parse_jobs_from_html(html)
1557
- logger.info(
1558
- f"parallel[{keyword[:30]}]: start={start} → {len(page_jobs)} jobs"
1559
- )
1560
-
1561
- if not page_jobs:
1562
- break
1563
-
1564
- collected.extend(page_jobs)
1565
- start += len(page_jobs)
1566
-
1567
- # Human-like delays between pages
1568
- await asyncio.sleep(random.uniform(4.0, 7.0))
1569
-
1570
- except Exception as e:
1571
- logger.warning(
1572
- f"parallel[{keyword[:30]}]: unexpected error: {e}; "
1573
- f"returning {len(collected)} jobs"
1574
- )
1575
-
1576
- # Cool-down before releasing semaphore to next keyword
1577
- await asyncio.sleep(random.uniform(4.0, 8.0))
1578
-
1579
- logger.info(f"parallel[{keyword[:30]}]: done — {len(collected)} jobs total")
1580
- return collected
1581
-
1582
-
1583
- def _resolve_search(user_keyword: str) -> tuple[list[str], Optional[str]]:
1584
- """Resolve user input into expanded search keywords + optional school filter.
1585
-
1586
- Resolution order:
1587
- 1. Exact school code (case-insensitive): "SOCSE" → all SOCSE terms
1588
- 2. Shortcut keyword: "software" → SOCSE terms
1589
- 3. Direct passthrough: "Data Analyst" → ["Data Analyst"], no filter
1590
-
1591
- Returns:
1592
- (search_keywords, school_code_to_filter_by_or_None)
1593
- """
1594
- key = user_keyword.strip().lower()
1595
-
1596
- # 1. Exact school code match (SOCSE, solas, SOB, etc.)
1597
- school_lower_map = {code.lower(): code for code in SCHOOLS}
1598
- if key in school_lower_map:
1599
- code = school_lower_map[key]
1600
- terms = SCHOOL_SEARCH_TERMS.get(key, [])
1601
- logger.info(
1602
- f"resolve_search: school code '{code}' → {len(terms)} search terms"
1603
- )
1604
- return terms if terms else [user_keyword], code
1605
-
1606
- # 2. Shortcut keyword → school (e.g. "software" → SOCSE)
1607
- if key in KEYWORD_TO_SCHOOL:
1608
- school_code = KEYWORD_TO_SCHOOL[key]
1609
- terms = []
1610
- for school in school_code.split(","):
1611
- for t in SCHOOL_SEARCH_TERMS.get(school.strip().lower(), []).copy():
1612
- if t not in terms:
1613
- terms.append(t)
1614
-
1615
- # Guarantee the user's explicit shortcut keyword is also searched!
1616
- # Sometimes shortcuts are exact job titles (e.g. "marketing", "data").
1617
- if key not in [t.lower() for t in terms]:
1618
- # Insert at the beginning so it's prioritized
1619
- terms.insert(0, user_keyword.strip())
1620
-
1621
- logger.info(
1622
- f"resolve_search: shortcut '{key}' → school {school_code} "
1623
- f"({len(terms)} search terms including shortcut)"
1624
- )
1625
- return terms if terms else [user_keyword], school_code
1626
-
1627
- # 3. Direct keyword — no expansion, no school filter
1628
- logger.info(f"resolve_search: direct keyword '{user_keyword}' (no expansion)")
1629
- return [user_keyword], None
1630
-
1631
-
1632
- def _deduplicate_jobs(jobs: list[dict]) -> tuple[list[dict], int]:
1633
- """Deduplicate jobs by canonical link. Returns (deduped_list, num_removed)."""
1634
- seen: dict[str, dict] = {}
1635
- untagged: list[dict] = []
1636
- for job in jobs:
1637
- key = job.get("link")
1638
- if key:
1639
- if key not in seen:
1640
- seen[key] = job
1641
- else:
1642
- untagged.append(job)
1643
- deduped = list(seen.values()) + untagged
1644
- return deduped, len(jobs) - len(deduped)
1645
-
1646
-
1647
-
1648
- @app.post("/scrape-internships")
1649
- async def scrape_internships(
1650
- keywords: str = Query("Software Engineer", description="Job search keywords"),
1651
- freshness: str = Query("r86400", description="Posting age: r3600=1h r86400=24h r604800=week r2592000=month"),
1652
- work_types: Optional[list[str]] = Query(
1653
- default=None,
1654
- description="Work-type checkboxes — repeat per selection: onsite | remote | hybrid",
1655
- ),
1656
- job_types: Optional[list[str]] = Query(
1657
- default=None,
1658
- description="Job-type checkboxes — repeat per selection: internship | full_time",
1659
- ),
1660
- ):
1661
- # ── Resolve keywords + optional school filter ──────────────
1662
- all_keywords, school_filter = _resolve_search(keywords)
1663
-
1664
- if job_types and any(jt.lower() == "internship" for jt in job_types):
1665
- all_keywords = [f"{kw} intern" for kw in all_keywords]
1666
- keywords = f"{keywords} intern"
1667
-
1668
- is_multi = len(all_keywords) > 1
1669
-
1670
- # Build the "primary" URL (used for the "Open in LinkedIn" link)
1671
- primary_url = build_search_url(
1672
- keywords=keywords,
1673
- freshness=freshness,
1674
- work_types=work_types,
1675
- job_types=job_types,
1676
- )
1677
- primary_params = build_search_params(
1678
- keywords=keywords,
1679
- freshness=freshness,
1680
- work_types=work_types,
1681
- job_types=job_types,
1682
- )
1683
-
1684
- async def _stream_generator():
1685
- """NDJSON streaming generator.
1686
-
1687
- Emits lines of JSON, one per event:
1688
- {"type": "start", ...}
1689
- {"type": "info", "message": ...}
1690
- {"type": "jobs", "data": [...], ...}
1691
- {"type": "done", ...}
1692
- {"type": "error", "message": ...}
1693
- """
1694
- all_jobs: list[dict] = []
1695
- engine_used = "http"
1696
- total_dupes_removed = 0
1697
- total_filtered_out = 0
1698
-
1699
- try:
1700
- # ── Emit start event ──────────────────────────────
1701
- yield json.dumps({
1702
- "type": "start",
1703
- "engine": "http",
1704
- "total_searches": len(all_keywords) if is_multi else 1,
1705
- }) + "\n"
1706
-
1707
- if is_multi:
1708
- school_label = f" (school: {school_filter})" if school_filter else ""
1709
- yield json.dumps({
1710
- "type": "info",
1711
- "message": f"Expanding '{keywords}' → {len(all_keywords)} parallel searches{school_label}",
1712
- }) + "\n"
1713
-
1714
- # ── Warm up cookies once ──────────────────────────
1715
- cookies = await _warm_up_cookies(force=False)
1716
- if not cookies:
1717
- yield json.dumps({
1718
- "type": "info",
1719
- "message": "Cookie warm-up: re-warming...",
1720
- }) + "\n"
1721
- cookies = await _warm_up_cookies(force=True)
1722
-
1723
- if is_multi:
1724
- # ══ PARALLEL MULTI-KEYWORD PATH ══════════════
1725
- # Fan out all keywords as concurrent tasks.
1726
- # As each completes, stream its results immediately.
1727
-
1728
- seen_links: set = set()
1729
-
1730
- async def _scrape_and_collect(kw: str, index: int):
1731
- """Scrape one keyword with a stagger delay to avoid 429s."""
1732
- # Stagger: each task waits before starting so requests are spread out.
1733
- stagger = index * random.uniform(3.0, 6.0)
1734
- if stagger > 0:
1735
- await asyncio.sleep(stagger)
1736
- jobs = await _fetch_all_pages_for_keyword(
1737
- keyword=kw,
1738
- base_params=primary_params,
1739
- cookies=cookies,
1740
- max_pages=10,
1741
- )
1742
- return kw, jobs
1743
-
1744
- # Create all tasks
1745
- tasks = [
1746
- asyncio.create_task(_scrape_and_collect(kw, i))
1747
- for i, kw in enumerate(all_keywords)
1748
- ]
1749
-
1750
- completed = 0
1751
- for coro in asyncio.as_completed(tasks):
1752
- try:
1753
- kw, kw_jobs = await coro
1754
- completed += 1
1755
-
1756
- # Deduplicate against already-seen links
1757
- new_jobs = []
1758
- for job in kw_jobs:
1759
- link = job.get("link")
1760
- if link:
1761
- if link not in seen_links:
1762
- seen_links.add(link)
1763
- new_jobs.append(job)
1764
- else:
1765
- total_dupes_removed += 1
1766
- else:
1767
- new_jobs.append(job)
1768
-
1769
- if school_filter and new_jobs:
1770
- pre = len(new_jobs)
1771
- filter_schools = [s.strip() for s in school_filter.split(",")]
1772
- new_jobs = [
1773
- j for j in new_jobs
1774
- if any(sf in j.get("schools", []) for sf in filter_schools)
1775
- ]
1776
- total_filtered_out += pre - len(new_jobs)
1777
-
1778
- # Strict internship title filter if only internship is selected
1779
- if job_types and "internship" in job_types and "full_time" not in job_types:
1780
- pre = len(new_jobs)
1781
- intern_kw = ["intern", "trainee", "student", "co-op", "apprentice", "fellow"]
1782
- new_jobs = [
1783
- j for j in new_jobs
1784
- if any(ik in j.get("title", "").lower() for ik in intern_kw)
1785
- ]
1786
- total_filtered_out += pre - len(new_jobs)
1787
-
1788
-
1789
- all_jobs.extend(new_jobs)
1790
-
1791
- logger.info(
1792
- f"multi-search [{completed}/{len(all_keywords)}]: "
1793
- f"'{kw}' → {len(kw_jobs)} raw, {len(new_jobs)} new "
1794
- f"(total {len(all_jobs)})"
1795
- )
1796
-
1797
- # Stream the batch
1798
- if new_jobs:
1799
- yield json.dumps({
1800
- "type": "jobs",
1801
- "data": new_jobs,
1802
- "keyword": kw,
1803
- "engine": "http",
1804
- "deduplicated": total_dupes_removed,
1805
- }) + "\n"
1806
-
1807
- yield json.dumps({
1808
- "type": "info",
1809
- "message": (
1810
- f"[{completed}/{len(all_keywords)}] "
1811
- f"'{kw}' → {len(new_jobs)} new jobs "
1812
- f"(total: {len(all_jobs)})"
1813
- ),
1814
- }) + "\n"
1815
-
1816
- except Exception as e:
1817
- completed += 1
1818
- logger.warning(f"multi-search: task error: {e}")
1819
- yield json.dumps({
1820
- "type": "info",
1821
- "message": f"[{completed}/{len(all_keywords)}] Search failed: {e}",
1822
- }) + "\n"
1823
-
1824
- engine_used = "http"
1825
-
1826
-
1827
- else:
1828
- # ══ SINGLE-KEYWORD PATH ══════════════════════
1829
- # Original flow: HTTP → Selenium → Playwright
1830
- jobs, engine_used = await _scrape_one_url(
1831
- primary_url, params=primary_params, paginate=True
1832
- )
1833
-
1834
- if jobs:
1835
- jobs, dupes = _deduplicate_jobs(jobs)
1836
- total_dupes_removed = dupes
1837
-
1838
- if school_filter:
1839
- pre = len(jobs)
1840
- filter_schools = [s.strip() for s in school_filter.split(",")]
1841
- jobs = [
1842
- j for j in jobs
1843
- if any(sf in j.get("schools", []) for sf in filter_schools)
1844
- ]
1845
- total_filtered_out += pre - len(jobs)
1846
-
1847
- # Strict internship title filter if only internship is selected
1848
- if job_types and "internship" in job_types and "full_time" not in job_types:
1849
- pre = len(jobs)
1850
- intern_kw = ["intern", "trainee", "student", "co-op", "apprentice", "fellow"]
1851
- jobs = [
1852
- j for j in jobs
1853
- if any(ik in j.get("title", "").lower() for ik in intern_kw)
1854
- ]
1855
- total_filtered_out += pre - len(jobs)
1856
-
1857
-
1858
- all_jobs = jobs
1859
-
1860
- yield json.dumps({
1861
- "type": "jobs",
1862
- "data": all_jobs,
1863
- "engine": engine_used,
1864
- "deduplicated": total_dupes_removed,
1865
- }) + "\n"
1866
-
1867
- # ── Emit done event ───────────────────────────────
1868
- if not all_jobs:
1869
- yield json.dumps({
1870
- "type": "error",
1871
- "message": "All scraping engines failed — 0 results",
1872
- }) + "\n"
1873
- else:
1874
- try:
1875
- logger.info(
1876
- f"serper: Searching hiring contacts for {len(all_jobs)} companies..."
1877
- )
1878
- yield json.dumps({
1879
- "type": "info",
1880
- "message": f"Searching for hiring contacts via Google for {len(all_jobs)} companies...",
1881
- }) + "\n"
1882
- await _fetch_hiring_contacts_via_serper(all_jobs)
1883
- found_count = sum(
1884
- 1 for j in all_jobs if j.get("contact_details")
1885
- )
1886
- logger.info(
1887
- f"serper: Hiring contact search done — "
1888
- f"{found_count}/{len(all_jobs)} jobs have a contact"
1889
- )
1890
- # ── VERIFICATION PRINT (visible in terminal logs) ──────
1891
- print(f"\n{'='*60}")
1892
- print(f"[SERPER HIRING CONTACTS] {found_count}/{len(all_jobs)} jobs enriched")
1893
- for j in all_jobs:
1894
- contacts = j.get("contact_details") or []
1895
- if contacts:
1896
- for c in contacts:
1897
- if isinstance(c, dict):
1898
- print(
1899
- f" ✓ {j.get('title','?')} @ {j.get('company','?')}: "
1900
- f"{c.get('name','?')} → {c.get('url','?')}"
1901
- )
1902
- else:
1903
- print(f" ✓ {j.get('title','?')} @ {j.get('company','?')}: {c}")
1904
- print(f"{'='*60}\n")
1905
-
1906
- # ── Stream enriched jobs back to frontend ──────────────
1907
- enriched_jobs = [j for j in all_jobs if j.get("contact_details")]
1908
- if enriched_jobs:
1909
- yield json.dumps({
1910
- "type": "jobs",
1911
- "data": enriched_jobs,
1912
- "engine": engine_used,
1913
- "deduplicated": 0,
1914
- "is_contact_update": True,
1915
- }) + "\n"
1916
- logger.info(
1917
- f"serper: Streamed {len(enriched_jobs)} enriched jobs back to frontend"
1918
- )
1919
- except Exception as e:
1920
- logger.warning(f"serper: Hiring contact search failed: {e}")
1921
-
1922
- # Save jobs to database for student dashboard
1923
- search_params = {
1924
- 'keywords': keywords,
1925
- 'freshness': freshness,
1926
- 'work_types': work_types or [],
1927
- 'job_types': job_types or []
1928
- }
1929
- try:
1930
- saved_count = save_scraped_jobs(all_jobs, search_params)
1931
- logger.info(f"Saved {saved_count} jobs to database")
1932
- except Exception as e:
1933
- logger.warning(f"Failed to save jobs to database: {e}")
1934
-
1935
- yield json.dumps({
1936
- "type": "done",
1937
- "total": len(all_jobs),
1938
- "engine": engine_used,
1939
- "url": primary_url,
1940
- "deduplicated": total_dupes_removed,
1941
- "filtered_out": total_filtered_out,
1942
- "school_filter": school_filter,
1943
- "searches_completed": len(all_keywords),
1944
- }) + "\n"
1945
-
1946
- except Exception as e:
1947
- logger.exception(f"scrape-internships stream error: {e}")
1948
- # If we have collected any jobs, emit them first with partial results notification
1949
- if all_jobs:
1950
- yield json.dumps({
1951
- "type": "info",
1952
- "message": f"Stream interrupted after collecting {len(all_jobs)} jobs",
1953
- }) + "\n"
1954
- yield json.dumps({
1955
- "type": "done",
1956
- "total": len(all_jobs),
1957
- "engine": engine_used,
1958
- "partial": True,
1959
- "error": str(e),
1960
- "url": primary_url,
1961
- "deduplicated": total_dupes_removed,
1962
- "filtered_out": total_filtered_out,
1963
- "school_filter": school_filter,
1964
- "searches_completed": len(all_keywords),
1965
- }) + "\n"
1966
- else:
1967
- # Only emit error if we have no jobs
1968
- yield json.dumps({
1969
- "type": "error",
1970
- "message": str(e),
1971
- }) + "\n"
1972
-
1973
- global global_scrape
1974
- if global_scrape.is_active:
1975
- global_scrape.cancel()
1976
-
1977
- session = ScrapeSession()
1978
- session.is_active = True
1979
- global_scrape = session
1980
-
1981
- async def _pump():
1982
- search_params_for_db = {
1983
- 'keywords': keywords,
1984
- 'freshness': freshness,
1985
- 'work_types': work_types or [],
1986
- 'job_types': job_types or []
1987
- }
1988
- try:
1989
- async for item in _stream_generator():
1990
- # Intercept and incrementally save jobs to DB so they appear in student portal mid-scrape
1991
- try:
1992
- data = json.loads(item.strip())
1993
- if data.get("type") == "jobs" and data.get("data"):
1994
- save_scraped_jobs(data["data"], search_params_for_db)
1995
- except Exception as e:
1996
- logger.warning(f"Incremental save failed: {e}")
1997
-
1998
- # Save to history and broadcast to active queues
1999
- session.history.append(item)
2000
- for q in session.queues:
2001
- await q.put(item)
2002
- except Exception as e:
2003
- logger.error(f"Background scraper error: {e}")
2004
- finally:
2005
- session.is_active = False
2006
- for q in session.queues:
2007
- await q.put(None)
2008
- session.queues.clear()
2009
-
2010
- # Start the background task so it continues even if the HTTP connection drops
2011
- session.task = asyncio.create_task(_pump())
2012
-
2013
- async def _consumer():
2014
- q = asyncio.Queue()
2015
- session.queues.append(q)
2016
- try:
2017
- while True:
2018
- item = await q.get()
2019
- if item is None:
2020
- break
2021
- yield item
2022
- except asyncio.CancelledError:
2023
- logger.info("Client disconnected, background scraper will continue.")
2024
- raise
2025
- finally:
2026
- if q in session.queues:
2027
- session.queues.remove(q)
2028
-
2029
- return StreamingResponse(
2030
- _consumer(),
2031
- media_type="application/x-ndjson",
2032
- headers={
2033
- "Cache-Control": "no-cache",
2034
- "X-Accel-Buffering": "no",
2035
- },
2036
- )
2037
-
2038
- @app.get("/scrape-internships/status")
2039
- async def scrape_status():
2040
- """Return whether a scrape session is currently active or has history."""
2041
- return {
2042
- "is_active": global_scrape.is_active,
2043
- "has_history": len(global_scrape.history) > 0
2044
- }
2045
-
2046
- @app.get("/scrape-internships/stream")
2047
- async def stream_scrape():
2048
- """Reconnect to the active scrape session or replay the last session's history."""
2049
- async def _reconnect_consumer():
2050
- # Yield all past events instantly
2051
- for item in global_scrape.history:
2052
- yield item
2053
-
2054
- # If the session is still active, listen for new events
2055
- if global_scrape.is_active:
2056
- q = asyncio.Queue()
2057
- global_scrape.queues.append(q)
2058
- try:
2059
- while True:
2060
- item = await q.get()
2061
- if item is None:
2062
- break
2063
- yield item
2064
- except asyncio.CancelledError:
2065
- raise
2066
- finally:
2067
- if q in global_scrape.queues:
2068
- global_scrape.queues.remove(q)
2069
-
2070
- return StreamingResponse(
2071
- _reconnect_consumer(),
2072
- media_type="application/x-ndjson",
2073
- headers={
2074
- "Cache-Control": "no-cache",
2075
- "X-Accel-Buffering": "no",
2076
- },
2077
- )
2078
-
2079
-
2080
- # ── Student Dashboard Endpoints ───────────────────────────────
2081
-
2082
- @app.get("/student/jobs/recent")
2083
- def get_recent_jobs(hours: int = Query(1, description="Hours ago (1 or 24)")):
2084
- """Get jobs scraped within the last N hours for student dashboard"""
2085
- try:
2086
- if hours == 1:
2087
- # Jobs from the last 1 hour
2088
- jobs = get_jobs_in_timeframe(1, 0)
2089
- elif hours == 24:
2090
- # Jobs from 1-24 hours ago (excluding the last hour to avoid overlap)
2091
- jobs = get_jobs_in_timeframe(24, 1)
2092
- else:
2093
- raise HTTPException(status_code=400, detail="Only 1 or 24 hours supported")
2094
-
2095
- return {
2096
- "jobs": jobs,
2097
- "count": len(jobs),
2098
- "timeframe": f"{hours} hour{'s' if hours > 1 else ''} ago"
2099
- }
2100
- except Exception as e:
2101
- raise HTTPException(status_code=500, detail=str(e))
2102
-
2103
- @app.get("/student/jobs/all-timeframes")
2104
- def get_all_timeframes(background_tasks: BackgroundTasks):
2105
- """Get jobs for all timeframes for student dashboard and auto-cleanup old jobs"""
2106
- try:
2107
- # Auto-delete jobs older than 30 days in the background
2108
- background_tasks.add_task(cleanup_old_jobs, 30)
2109
-
2110
- # Fetch all jobs in one query and bin them to save DB roundtrips
2111
- return get_binned_jobs()
2112
- except Exception as e:
2113
- raise HTTPException(status_code=500, detail=str(e))
2114
-
2115
- @app.post("/admin/cleanup")
2116
- def cleanup_old_data(days: int = Query(7, description="Remove jobs older than N days")):
2117
- """Admin endpoint to cleanup old job data"""
2118
- try:
2119
- deleted_count = cleanup_old_jobs(days)
2120
- return {
2121
- "message": f"Cleaned up {deleted_count} jobs older than {days} days",
2122
- "deleted_count": deleted_count
2123
- }
2124
- except Exception as e:
2125
- raise HTTPException(status_code=500, detail=str(e))
2126
-
2127
-
2128
- @app.post("/admin/trigger-scrape")
2129
- async def trigger_auto_scrape():
2130
- """
2131
- Manually kick off the full school sweep. Returns immediately;
2132
- progress can be polled at GET /admin/scrape-status.
2133
- If a sweep is already running, returns 409.
2134
- """
2135
- global _auto_scrape_task
2136
- if _auto_scrape_state.get("is_running"):
2137
- return {
2138
- "status": "already_running",
2139
- "message": "Auto-scrape is already in progress.",
2140
- "state": _auto_scrape_state,
2141
- }
2142
-
2143
- scheduler.add_job(
2144
- _auto_scrape_all_schools,
2145
- trigger=IntervalTrigger(hours=1),
2146
- id="hourly_school_scrape",
2147
- replace_existing=True,
2148
- next_run_time=datetime.now(timezone.utc),
2149
- misfire_grace_time=300,
2150
- )
2151
-
2152
- return {
2153
- "status": "started",
2154
- "message": f"Auto-scrape started for {len(_AUTO_SCRAPE_SCHOOLS)} schools (running hourly).",
2155
- "schools": _AUTO_SCRAPE_SCHOOLS,
2156
- }
2157
-
2158
-
2159
- @app.post("/admin/stop-scrape")
2160
- async def stop_auto_scrape():
2161
- """
2162
- Stop the currently running auto-scrape sweep and remove the hourly scheduled job.
2163
- """
2164
- if scheduler.get_job("hourly_school_scrape"):
2165
- scheduler.remove_job("hourly_school_scrape")
2166
-
2167
- _auto_scrape_state["cancel_requested"] = True
2168
-
2169
- global global_scrape
2170
- if global_scrape.is_active:
2171
- global_scrape.cancel()
2172
-
2173
- return {"status": "stopped", "message": "Auto-scrape has been stopped and scheduled jobs removed."}
2174
-
2175
-
2176
- @app.get("/admin/scrape-status")
2177
- async def get_scrape_status():
2178
- """Poll the live progress of the currently-running (or last) auto-scrape sweep."""
2179
- state = dict(_auto_scrape_state)
2180
- if state.get("started_at"):
2181
- elapsed = (state.get("finished_at") or time.time()) - state["started_at"]
2182
- state["elapsed_seconds"] = round(elapsed)
2183
- total_done = len(state.get("completed", [])) + len(state.get("failed", []))
2184
- state["progress"] = f"{total_done}/{len(_AUTO_SCRAPE_SCHOOLS)}"
2185
- return state
2186
-
2187
-
2188
- @app.get("/admin/scrape-history")
2189
- async def get_scrape_history():
2190
- """Return the full history of all auto-scrape sweep runs (newest first).
2191
-
2192
- Each entry has: sweep_id, is_running, current_school, completed, failed,
2193
- started_at, finished_at, cancelled, elapsed_seconds, progress.
2194
- """
2195
- history = []
2196
- for entry in reversed(_auto_scrape_history):
2197
- item = dict(entry)
2198
- # Compute elapsed seconds
2199
- if item.get("started_at"):
2200
- elapsed = (item.get("finished_at") or time.time()) - item["started_at"]
2201
- item["elapsed_seconds"] = round(elapsed)
2202
- # Compute progress string
2203
- total_done = len(item.get("completed", [])) + len(item.get("failed", []))
2204
- item["progress"] = f"{total_done}/{len(_AUTO_SCRAPE_SCHOOLS)}"
2205
- item["total_schools"] = len(_AUTO_SCRAPE_SCHOOLS)
2206
- item["schools"] = _AUTO_SCRAPE_SCHOOLS
2207
- history.append(item)
2208
- return {"history": history, "total_runs": len(history)}
2209
-
2210
-
2211
- async def process_company_ratings(companies: list[str]) -> dict:
2212
- if not companies:
2213
- return {}
2214
-
2215
- SERPER_API_KEY = os.environ.get("SERPER_API_KEY", "0820e51e9080c289b3849e30189e023774fdad87")
2216
- ratings = {}
2217
-
2218
- async def _fetch_company_rating(client: httpx.AsyncClient, company: str, semaphore: asyncio.Semaphore) -> tuple[str, float]:
2219
- async with semaphore:
2220
- await asyncio.sleep(random.uniform(0.5, 1.5))
2221
- query = f'"{company}" reviews Bangalore site:glassdoor.co.in OR site:ambitionbox.com'
2222
- try:
2223
- resp = await client.post(
2224
- "https://google.serper.dev/search",
2225
- json={"q": query},
2226
- headers={"X-API-KEY": SERPER_API_KEY, "Content-Type": "application/json"}
2227
- )
2228
- data = resp.json()
2229
- for org in data.get("organic", []):
2230
- # Try Serper's built-in rating first
2231
- if "rating" in org and isinstance(org["rating"], (int, float)):
2232
- return company, float(org["rating"])
2233
- # Fallback to regex on snippet
2234
- snippet = org.get("snippet", "")
2235
- match = re.search(r"Rating:\s*([\d\.]+)", snippet, re.IGNORECASE)
2236
- if match:
2237
- return company, float(match.group(1))
2238
- except Exception as e:
2239
- logger.warning(f"rate-companies: Serper search failed for {company}: {repr(e)}")
2240
- return company, 2.5
2241
-
2242
- try:
2243
- # We limit concurrency to 3 to avoid overwhelming the network or API limits
2244
- semaphore = asyncio.Semaphore(3)
2245
- async with httpx.AsyncClient(timeout=15.0) as client:
2246
- tasks = [_fetch_company_rating(client, c, semaphore) for c in companies]
2247
- results = await asyncio.gather(*tasks)
2248
-
2249
- for company, rating in results:
2250
- # Clamp to [1, 5] just in case
2251
- ratings[company] = round(max(1.0, min(5.0, rating)), 1)
2252
-
2253
- # Fill in any companies missed with a neutral score
2254
- for company in companies:
2255
- if company not in ratings:
2256
- ratings[company] = 2.5
2257
-
2258
- return ratings
2259
- except Exception as e:
2260
- logger.error(f"rate-companies: process_company_ratings failed: {repr(e)}")
2261
- raise e
2262
-
2263
-
2264
- class RateCompaniesRequest(BaseModel):
2265
- companies: list[str]
2266
-
2267
-
2268
- @app.post("/rate-companies")
2269
- async def rate_companies(req: RateCompaniesRequest, background_tasks: BackgroundTasks):
2270
- """
2271
- Use Serper.dev (Google Search API) to estimate a reputation/internship
2272
- quality rating (1.0–5.0) for each submitted company name (Bangalore context).
2273
-
2274
- A concurrent search is made for each company, extracting the rating from
2275
- Glassdoor/Ambitionbox search snippets.
2276
- """
2277
- try:
2278
- ratings = await process_company_ratings(req.companies)
2279
- # Persist ratings to DB in the background (non-blocking)
2280
- background_tasks.add_task(_persist_ratings, ratings)
2281
-
2282
- logger.info(f"rate-companies: successfully rated {len(ratings)} companies via Search API")
2283
- return {"ratings": ratings}
2284
-
2285
- except Exception as e:
2286
- logger.error(f"rate-companies: Search API call failed: {repr(e)}")
2287
- raise HTTPException(status_code=500, detail=f"Search rating failed: {str(e)}")
2288
-
2289
-
2290
- def _persist_ratings(ratings: dict):
2291
- """Write company ratings back to the DB (called as a background task)."""
2292
- try:
2293
- updated = update_company_ratings(ratings)
2294
- logger.info(f"rate-companies: persisted ratings for {updated} job rows in DB")
2295
- except Exception as e:
2296
- logger.error(f"rate-companies: DB persist failed: {e}")
2297
-
2298
-
2299
- # ── Manual Rating Management ─────────────────────────────────────
2300
-
2301
- class SetRatingRequest(BaseModel):
2302
- company: str
2303
- rating: float
2304
-
2305
- @app.post("/set-company-rating")
2306
- async def set_company_rating(req: SetRatingRequest):
2307
- """Manually set a rating for a specific company (for testing/admin purposes)."""
2308
- try:
2309
- # Validate rating range
2310
- rating = max(1.0, min(5.0, req.rating))
2311
-
2312
- # Update the database
2313
- updated = update_company_ratings({req.company: rating})
2314
-
2315
- return {
2316
- "message": f"Updated rating for '{req.company}' to {rating}",
2317
- "company": req.company,
2318
- "rating": rating,
2319
- "rows_updated": updated
2320
- }
2321
- except Exception as e:
2322
- logger.error(f"set-company-rating: failed: {e}")
2323
- raise HTTPException(status_code=500, detail=str(e))
2324
-
2325
-
2326
- @app.get("/get-company-ratings")
2327
- async def get_company_ratings():
2328
- """Get all companies and their current ratings from the database."""
2329
- try:
2330
- conn = _connect()
2331
- try:
2332
- with conn.cursor() as cur:
2333
- cur.execute(
2334
- """
2335
- SELECT DISTINCT company, company_rating
2336
- FROM scraped_jobs
2337
- WHERE company IS NOT NULL AND company_rating IS NOT NULL
2338
- ORDER BY company
2339
- """
2340
- )
2341
- rows = cur.fetchall()
2342
- finally:
2343
- conn.close()
2344
-
2345
- ratings = {row[0]: row[1] for row in rows}
2346
- return {"ratings": ratings}
2347
- except Exception as e:
2348
- logger.error(f"get-company-ratings: failed: {e}")
2349
- raise HTTPException(status_code=500, detail=str(e))
2350
-