import { desc, lt, sql } from "drizzle-orm"; import { db } from "@/db"; import { jobs, syncRuns } from "@/db/schema"; import { SOURCES, type NJob } from "./sources"; const g = globalThis as typeof globalThis & { __hfSync?: Promise | null }; export type SyncSummary = { fetched: number; inserted: number; sources: { key: string; fetched: number; inserted: number; error?: string }[] }; async function upsert(list: NJob[]) { const seen = new Set(); const cutoff = Date.now() - 44 * 86400_000; const rows = list.filter((j) => { const k = `${j.source}:${j.externalId}`; if (seen.has(k) || !j.title || !j.url) return false; if (!isNaN(j.postedAt.getTime()) && j.postedAt.getTime() < cutoff) return false; seen.add(k); return true; }); for (let i = 0; i < rows.length; i += 100) { const safe = (n: number | null) => (n != null && Number.isFinite(n) && n > 0 ? Math.min(Math.round(n), 2_000_000_000) : null); const chunk = rows.slice(i, i + 100).map((j) => ({ ...j, title: j.title.replace(/\u0000/g, "").slice(0, 300), description: j.description.replace(/\u0000/g, ""), salaryMin: safe(j.salaryMin), salaryMax: safe(j.salaryMax), postedAt: isNaN(j.postedAt.getTime()) ? new Date() : j.postedAt, fetchedAt: new Date(), })); await db .insert(jobs) .values(chunk) .onConflictDoUpdate({ target: [jobs.source, jobs.externalId], set: { title: sql`excluded.title`, description: sql`excluded.description`, tags: sql`excluded.tags`, salaryMin: sql`excluded.salary_min`, salaryMax: sql`excluded.salary_max`, fetchedAt: sql`excluded.fetched_at`, }, }); } return rows.length; } async function countSource(key: string) { const r = await db.execute(sql`select count(*)::int as c from jobs where source = ${key}`); return (r.rows[0] as { c: number }).c; } async function run(): Promise { const results = await Promise.all( SOURCES.map(async (s) => { const t0 = Date.now(); try { const before = await countSource(s.key); const list = await s.fetch(); const n = await upsert(list); const after = await countSource(s.key); const inserted = Math.max(0, after - before); await db.insert(syncRuns).values({ source: s.key, fetched: n, inserted, status: "ok", durationMs: Date.now() - t0 }); return { key: s.key, fetched: n, inserted }; } catch (e) { const cause = (e as { cause?: { message?: string } })?.cause?.message; const error = cause || (e instanceof Error ? e.message : String(e)); await db.insert(syncRuns).values({ source: s.key, status: "error", error: error.slice(0, 200), durationMs: Date.now() - t0 }); return { key: s.key, fetched: 0, inserted: 0, error }; } }), ); await db.delete(jobs).where(lt(jobs.postedAt, new Date(Date.now() - 45 * 86400_000))); return { fetched: results.reduce((a, r) => a + r.fetched, 0), inserted: results.reduce((a, r) => a + r.inserted, 0), sources: results, }; } export function syncAllSources(): Promise { if (!g.__hfSync) { g.__hfSync = run().finally(() => { g.__hfSync = null; }); } return g.__hfSync; } export async function lastSyncAt(): Promise { const r = await db.select({ at: syncRuns.startedAt }).from(syncRuns).orderBy(desc(syncRuns.startedAt)).limit(1); return r[0]?.at ?? null; } export async function jobCount() { const r = await db.execute(sql`select count(*)::int as c from jobs`); return (r.rows[0] as { c: number }).c; }