add WorkingNomads + SkipTheDrive scrapers (social media, CS, entry-level focus)
Browse filesWorkingNomads: JSON API endpoint /api/exposed_jobs/, word-level keyword
matching against title/tags/category, returns up to 25 relevant remote jobs.
SkipTheDrive: HTML scraper with /?s={query} search URL, uses precise CSS
selectors (custom_fields_company_name_display_search_results, article.post
cards). Returns up to 25 results per query.
Both fill a gap in social media, customer service, and entry-level roles
that existing boards don't cover well. No freshness filter applied (empty
posted_date) since aggregator boards post jobs that stay live for weeks;
DB-level 7-day cleanup handles staleness.
- backend/config.py +10 -0
- backend/scrapers/international.py +63 -0
backend/config.py
CHANGED
|
@@ -305,6 +305,16 @@ JOB_BOARDS = {
|
|
| 305 |
"search_url": "https://blockchain.work/jobs?q={query}",
|
| 306 |
"type": "public", "region": "Remote/Web3", "icon": "global", "enabled": True,
|
| 307 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 308 |
},
|
| 309 |
}
|
| 310 |
|
|
|
|
| 305 |
"search_url": "https://blockchain.work/jobs?q={query}",
|
| 306 |
"type": "public", "region": "Remote/Web3", "icon": "global", "enabled": True,
|
| 307 |
},
|
| 308 |
+
"workingnomads": {
|
| 309 |
+
"base_url": "https://www.workingnomads.com",
|
| 310 |
+
"search_url": "https://www.workingnomads.com/api/exposed_jobs/",
|
| 311 |
+
"type": "public", "region": "Remote/Global", "icon": "global", "enabled": True,
|
| 312 |
+
},
|
| 313 |
+
"skipthedrive": {
|
| 314 |
+
"base_url": "https://www.skipthedrive.com",
|
| 315 |
+
"search_url": "https://www.skipthedrive.com/?s={query}",
|
| 316 |
+
"type": "public", "region": "Remote/Global", "icon": "global", "enabled": True,
|
| 317 |
+
},
|
| 318 |
},
|
| 319 |
}
|
| 320 |
|
backend/scrapers/international.py
CHANGED
|
@@ -324,6 +324,69 @@ class InternationalJobScraper(BaseScraper):
|
|
| 324 |
return self._extract_from_text(soup, "remoteworkng", "https://remotework.ng", CAT)
|
| 325 |
return result
|
| 326 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 327 |
# ββ Playwright scrapers (JS-heavy / login sites) βββββββββββββββββββββββ
|
| 328 |
|
| 329 |
async def scrape_indeed_playwright(self, page, query: str) -> list[dict]:
|
|
|
|
| 324 |
return self._extract_from_text(soup, "remoteworkng", "https://remotework.ng", CAT)
|
| 325 |
return result
|
| 326 |
|
| 327 |
+
# ββ WorkingNomads β JSON API ββββββββββββββββββββββββββββββββββββββββββββ
|
| 328 |
+
def scrape_workingnomads(self, query: str) -> list[dict]:
|
| 329 |
+
data = self.fetch_json("https://www.workingnomads.com/api/exposed_jobs/")
|
| 330 |
+
if not isinstance(data, list):
|
| 331 |
+
return []
|
| 332 |
+
q_words = set(query.lower().split())
|
| 333 |
+
jobs = []
|
| 334 |
+
for item in data:
|
| 335 |
+
title = item.get("title", "")
|
| 336 |
+
combined = (
|
| 337 |
+
title.lower() + " "
|
| 338 |
+
+ (item.get("tags", "") or "").lower() + " "
|
| 339 |
+
+ (item.get("category_name", "") or "").lower()
|
| 340 |
+
)
|
| 341 |
+
if not title or not q_words & set(combined.split()):
|
| 342 |
+
continue
|
| 343 |
+
url_val = item.get("url", "")
|
| 344 |
+
if not url_val:
|
| 345 |
+
continue
|
| 346 |
+
jobs.append(self._make_job(
|
| 347 |
+
title, item.get("company_name", ""),
|
| 348 |
+
item.get("location", "Remote"),
|
| 349 |
+
re.sub(r"<[^>]+>", "", (item.get("description", "") or ""))[:400],
|
| 350 |
+
url_val, "workingnomads", CAT,
|
| 351 |
+
"",
|
| 352 |
+
))
|
| 353 |
+
return jobs[:25]
|
| 354 |
+
|
| 355 |
+
# ββ SkipTheDrive β HTML ββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 356 |
+
def scrape_skipthedrive(self, query: str) -> list[dict]:
|
| 357 |
+
url = f"https://www.skipthedrive.com/?s={quote(query)}"
|
| 358 |
+
soup = self.fetch_soup(url)
|
| 359 |
+
if not soup:
|
| 360 |
+
return []
|
| 361 |
+
ld = self.fetch_json_ld(soup)
|
| 362 |
+
if ld:
|
| 363 |
+
for j in ld: j.update({"source": "skipthedrive", "category": CAT})
|
| 364 |
+
return ld[:25]
|
| 365 |
+
cards = self.multi_select(soup, [
|
| 366 |
+
"article.post","article.job","div.job-listing","li.job-listing",
|
| 367 |
+
"div[class*='job-card']","article","li",
|
| 368 |
+
])
|
| 369 |
+
jobs, seen = [], set()
|
| 370 |
+
for card in cards[:25]:
|
| 371 |
+
try:
|
| 372 |
+
title = self.first_text(card, ["h2 a","h3 a","a[class*='title']","strong a","a"])
|
| 373 |
+
href = self.first_href(card, ["h2 a","h3 a","a[class*='title']","strong a","a"], "https://www.skipthedrive.com")
|
| 374 |
+
if not title or not href or href in seen:
|
| 375 |
+
continue
|
| 376 |
+
seen.add(href)
|
| 377 |
+
company = self.first_text(card, [
|
| 378 |
+
"span.custom_fields_company_name_display_search_results",
|
| 379 |
+
"[class*='company_name']","[class*='company']","strong",
|
| 380 |
+
])
|
| 381 |
+
location = self.first_text(card, _LOCATION_SELS, "Remote")
|
| 382 |
+
desc = self.first_text(card, _DESC_SELS)
|
| 383 |
+
jobs.append(self._make_job(title, company, location, desc, href, "skipthedrive", CAT, ""))
|
| 384 |
+
except Exception:
|
| 385 |
+
continue
|
| 386 |
+
if not jobs:
|
| 387 |
+
return self._extract_from_text(soup, "skipthedrive", "https://www.skipthedrive.com", CAT)
|
| 388 |
+
return jobs[:25]
|
| 389 |
+
|
| 390 |
# ββ Playwright scrapers (JS-heavy / login sites) βββββββββββββββββββββββ
|
| 391 |
|
| 392 |
async def scrape_indeed_playwright(self, page, query: str) -> list[dict]:
|