api / src /lib /sources.ts
lljz66's picture
Put src/lib/sources.ts
cbf83bb verified
Raw History Blame Contribute Delete
7.94 kB
import { stripHtml } from "./utils";
export type NJob = {
source: string;
externalId: string;
title: string;
company: string;
companyLogo: string;
location: string;
remote: boolean;
jobType: string;
category: string;
salaryMin: number | null;
salaryMax: number | null;
salaryText: string;
tags: string[];
description: string;
url: string;
postedAt: Date;
};
const UA = { "User-Agent": "HuntFlow/1.0 (job aggregator; contact: hello@huntflow.app)" };
async function getJson<T>(url: string, timeout = 20000): Promise<T> {
const res = await fetch(url, {
headers: { ...UA, Accept: "application/json" },
signal: AbortSignal.timeout(timeout),
cache: "no-store",
});
if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
return (await res.json()) as T;
}
function clean(html: string) {
return stripHtml(html || "").slice(0, 9000);
}
function parseSalary(text: string): [number | null, number | null] {
const nums = [...text.matchAll(/(\d[\d,.]*)\s*(k)?/gi)]
.map((m) => {
let n = parseFloat(m[1].replace(/,/g, ""));
if (m[2]) n *= 1000;
return n;
})
.filter((n) => n >= 1000);
if (nums.length >= 2) return [Math.round(nums[0]), Math.round(nums[1])];
if (nums.length === 1) return [Math.round(nums[0]), null];
return [null, null];
}
type Remotive = {
jobs: {
id: number;
url: string;
title: string;
company_name: string;
company_logo: string;
category: string;
tags: string[];
job_type: string;
publication_date: string;
candidate_required_location: string;
salary: string;
description: string;
}[];
};
export async function fetchRemotive(): Promise<NJob[]> {
const data = await getJson<Remotive>("https://remotive.com/api/remote-jobs");
return data.jobs.map((j) => {
const [min, max] = parseSalary(j.salary || "");
return {
source: "remotive",
externalId: String(j.id),
title: j.title,
company: j.company_name.trim(),
companyLogo: j.company_logo || "",
location: j.candidate_required_location || "Worldwide",
remote: true,
jobType: (j.job_type || "").replace("_", " "),
category: j.category || "",
salaryMin: min,
salaryMax: max,
salaryText: j.salary || "",
tags: (j.tags || []).slice(0, 12),
description: clean(j.description),
url: j.url,
postedAt: new Date(j.publication_date + "Z"),
};
});
}
type Arbeit = {
data: {
slug: string;
company_name: string;
title: string;
description: string;
remote: boolean;
url: string;
tags: string[];
job_types: string[];
location: string;
created_at: number;
}[];
};
export async function fetchArbeitnow(): Promise<NJob[]> {
const pages = await Promise.allSettled(
[1, 2, 3].map((p) => getJson<Arbeit>(`https://www.arbeitnow.com/api/job-board-api?page=${p}`)),
);
const out: NJob[] = [];
for (const p of pages) {
if (p.status !== "fulfilled") continue;
for (const j of p.value.data) {
out.push({
source: "arbeitnow",
externalId: j.slug,
title: j.title,
company: j.company_name,
companyLogo: "",
location: j.remote ? `${j.location || "Europe"} · Remote` : j.location,
remote: j.remote,
jobType: (j.job_types?.[0] || "").replace(/ \/.*/, ""),
category: j.tags?.[0] || "",
salaryMin: null,
salaryMax: null,
salaryText: "",
tags: (j.tags || []).slice(0, 12),
description: clean(j.description),
url: j.url,
postedAt: new Date(j.created_at * 1000),
});
}
}
if (!out.length) throw new Error("no jobs returned");
return out;
}
type RemoteOk = {
slug?: string;
id?: string;
epoch?: number;
date?: string;
company?: string;
company_logo?: string;
logo?: string;
position?: string;
tags?: string[];
description?: string;
location?: string;
salary_min?: number;
salary_max?: number;
apply_url?: string;
url?: string;
};
export async function fetchRemoteOk(): Promise<NJob[]> {
const data = await getJson<RemoteOk[]>("https://remoteok.com/api");
return data
.filter((j) => j.position && j.company && j.id)
.map((j) => ({
source: "remoteok",
externalId: String(j.id),
title: j.position as string,
company: j.company as string,
companyLogo: j.company_logo || j.logo || "",
location: j.location || "Worldwide",
remote: true,
jobType: "full time",
category: j.tags?.[0] || "",
salaryMin: j.salary_min && j.salary_min > 0 ? j.salary_min : null,
salaryMax: j.salary_max && j.salary_max > 0 ? j.salary_max : null,
salaryText: "",
tags: (j.tags || []).slice(0, 12),
description: clean(j.description || ""),
url: j.url || j.apply_url || "https://remoteok.com",
postedAt: j.epoch ? new Date(j.epoch * 1000) : new Date(j.date || Date.now()),
}));
}
type Himalayas = {
nextCursor?: string | null;
jobs: {
title: string;
excerpt?: string;
companyName: string;
companyLogo?: string;
employmentType?: string;
minSalary?: number | null;
maxSalary?: number | null;
currency?: string | null;
salaryPeriod?: string | null;
seniority?: string[];
locationRestrictions?: string[];
categories?: string[];
parentCategories?: string[];
description?: string;
pubDate: number;
applicationLink: string;
guid: string;
}[];
};
export async function fetchHimalayas(): Promise<NJob[]> {
const out: NJob[] = [];
let cursor: string | null | undefined = undefined;
for (let i = 0; i < 6; i++) {
const url: string =
"https://himalayas.app/jobs/api?limit=20" + (cursor ? `&cursor=${encodeURIComponent(cursor)}` : "");
let data: Himalayas;
try {
data = await getJson<Himalayas>(url);
} catch (e) {
if (!out.length) throw e;
break;
}
for (const j of data.jobs) {
const locs = j.locationRestrictions?.length ? j.locationRestrictions.join(", ") : "Worldwide";
const mult = ({ hourly: 2080, hour: 2080, daily: 260, weekly: 52, monthly: 12 } as Record<string, number>)[(j.salaryPeriod || "annual").toLowerCase()] ?? 1;
const cur = (j.currency || "USD").toUpperCase();
const usdLike = cur === "USD";
const lo = j.minSalary ? Math.round(j.minSalary * mult) : null;
const hi = j.maxSalary ? Math.round(j.maxSalary * mult) : null;
const salaryText = !usdLike && (lo || hi) ? `${cur} ${lo ? Math.round(lo / 1000) + "k" : ""}${lo && hi ? " – " : ""}${hi ? Math.round(hi / 1000) + "k" : ""}`.trim() : "";
out.push({
source: "himalayas",
externalId: j.guid,
title: j.title,
company: j.companyName,
companyLogo: j.companyLogo || "",
location: locs,
remote: true,
jobType: (j.employmentType || "").toLowerCase(),
category: j.parentCategories?.[0] || j.categories?.[0] || "",
salaryMin: usdLike ? lo : null,
salaryMax: usdLike ? hi : null,
salaryText,
tags: [...(j.categories || []), ...(j.seniority || [])].map((t) => t.replace(/-/g, " ")).slice(0, 12),
description: clean(j.description || j.excerpt || ""),
url: j.applicationLink,
postedAt: new Date(j.pubDate * 1000),
});
}
cursor = data.nextCursor;
if (!cursor) break;
}
return out;
}
export const SOURCES: { key: string; label: string; fetch: () => Promise<NJob[]> }[] = [
{ key: "remotive", label: "Remotive", fetch: fetchRemotive },
{ key: "arbeitnow", label: "Arbeitnow", fetch: fetchArbeitnow },
{ key: "remoteok", label: "RemoteOK", fetch: fetchRemoteOk },
{ key: "himalayas", label: "Himalayas", fetch: fetchHimalayas },
];
export const SOURCE_LABEL: Record<string, string> = Object.fromEntries(
SOURCES.map((s) => [s.key, s.label]),
);