Britzzy commited on
Commit
c4a01a0
Β·
1 Parent(s): f8e5706

add WorkingNomads + SkipTheDrive scrapers (social media, CS, entry-level focus)

Browse files

WorkingNomads: JSON API endpoint /api/exposed_jobs/, word-level keyword
matching against title/tags/category, returns up to 25 relevant remote jobs.

SkipTheDrive: HTML scraper with /?s={query} search URL, uses precise CSS
selectors (custom_fields_company_name_display_search_results, article.post
cards). Returns up to 25 results per query.

Both fill a gap in social media, customer service, and entry-level roles
that existing boards don't cover well. No freshness filter applied (empty
posted_date) since aggregator boards post jobs that stay live for weeks;
DB-level 7-day cleanup handles staleness.

backend/config.py CHANGED
@@ -305,6 +305,16 @@ JOB_BOARDS = {
305
  "search_url": "https://blockchain.work/jobs?q={query}",
306
  "type": "public", "region": "Remote/Web3", "icon": "global", "enabled": True,
307
  },
 
 
 
 
 
 
 
 
 
 
308
  },
309
  }
310
 
 
305
  "search_url": "https://blockchain.work/jobs?q={query}",
306
  "type": "public", "region": "Remote/Web3", "icon": "global", "enabled": True,
307
  },
308
+ "workingnomads": {
309
+ "base_url": "https://www.workingnomads.com",
310
+ "search_url": "https://www.workingnomads.com/api/exposed_jobs/",
311
+ "type": "public", "region": "Remote/Global", "icon": "global", "enabled": True,
312
+ },
313
+ "skipthedrive": {
314
+ "base_url": "https://www.skipthedrive.com",
315
+ "search_url": "https://www.skipthedrive.com/?s={query}",
316
+ "type": "public", "region": "Remote/Global", "icon": "global", "enabled": True,
317
+ },
318
  },
319
  }
320
 
backend/scrapers/international.py CHANGED
@@ -324,6 +324,69 @@ class InternationalJobScraper(BaseScraper):
324
  return self._extract_from_text(soup, "remoteworkng", "https://remotework.ng", CAT)
325
  return result
326
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
327
  # ── Playwright scrapers (JS-heavy / login sites) ───────────────────────
328
 
329
  async def scrape_indeed_playwright(self, page, query: str) -> list[dict]:
 
324
  return self._extract_from_text(soup, "remoteworkng", "https://remotework.ng", CAT)
325
  return result
326
 
327
+ # ── WorkingNomads β€” JSON API ────────────────────────────────────────────
328
+ def scrape_workingnomads(self, query: str) -> list[dict]:
329
+ data = self.fetch_json("https://www.workingnomads.com/api/exposed_jobs/")
330
+ if not isinstance(data, list):
331
+ return []
332
+ q_words = set(query.lower().split())
333
+ jobs = []
334
+ for item in data:
335
+ title = item.get("title", "")
336
+ combined = (
337
+ title.lower() + " "
338
+ + (item.get("tags", "") or "").lower() + " "
339
+ + (item.get("category_name", "") or "").lower()
340
+ )
341
+ if not title or not q_words & set(combined.split()):
342
+ continue
343
+ url_val = item.get("url", "")
344
+ if not url_val:
345
+ continue
346
+ jobs.append(self._make_job(
347
+ title, item.get("company_name", ""),
348
+ item.get("location", "Remote"),
349
+ re.sub(r"<[^>]+>", "", (item.get("description", "") or ""))[:400],
350
+ url_val, "workingnomads", CAT,
351
+ "",
352
+ ))
353
+ return jobs[:25]
354
+
355
+ # ── SkipTheDrive β€” HTML ──────────────────────────────────────────────────
356
+ def scrape_skipthedrive(self, query: str) -> list[dict]:
357
+ url = f"https://www.skipthedrive.com/?s={quote(query)}"
358
+ soup = self.fetch_soup(url)
359
+ if not soup:
360
+ return []
361
+ ld = self.fetch_json_ld(soup)
362
+ if ld:
363
+ for j in ld: j.update({"source": "skipthedrive", "category": CAT})
364
+ return ld[:25]
365
+ cards = self.multi_select(soup, [
366
+ "article.post","article.job","div.job-listing","li.job-listing",
367
+ "div[class*='job-card']","article","li",
368
+ ])
369
+ jobs, seen = [], set()
370
+ for card in cards[:25]:
371
+ try:
372
+ title = self.first_text(card, ["h2 a","h3 a","a[class*='title']","strong a","a"])
373
+ href = self.first_href(card, ["h2 a","h3 a","a[class*='title']","strong a","a"], "https://www.skipthedrive.com")
374
+ if not title or not href or href in seen:
375
+ continue
376
+ seen.add(href)
377
+ company = self.first_text(card, [
378
+ "span.custom_fields_company_name_display_search_results",
379
+ "[class*='company_name']","[class*='company']","strong",
380
+ ])
381
+ location = self.first_text(card, _LOCATION_SELS, "Remote")
382
+ desc = self.first_text(card, _DESC_SELS)
383
+ jobs.append(self._make_job(title, company, location, desc, href, "skipthedrive", CAT, ""))
384
+ except Exception:
385
+ continue
386
+ if not jobs:
387
+ return self._extract_from_text(soup, "skipthedrive", "https://www.skipthedrive.com", CAT)
388
+ return jobs[:25]
389
+
390
  # ── Playwright scrapers (JS-heavy / login sites) ───────────────────────
391
 
392
  async def scrape_indeed_playwright(self, page, query: str) -> list[dict]: