api / src /lib /sync.ts
lljz66's picture
Put src/lib/sync.ts
05950f5 verified
Raw History Blame Contribute Delete
3.7 kB
import { desc, lt, sql } from "drizzle-orm";
import { db } from "@/db";
import { jobs, syncRuns } from "@/db/schema";
import { SOURCES, type NJob } from "./sources";
const g = globalThis as typeof globalThis & { __hfSync?: Promise<SyncSummary> | null };
export type SyncSummary = { fetched: number; inserted: number; sources: { key: string; fetched: number; inserted: number; error?: string }[] };
async function upsert(list: NJob[]) {
const seen = new Set<string>();
const cutoff = Date.now() - 44 * 86400_000;
const rows = list.filter((j) => {
const k = `${j.source}:${j.externalId}`;
if (seen.has(k) || !j.title || !j.url) return false;
if (!isNaN(j.postedAt.getTime()) && j.postedAt.getTime() < cutoff) return false;
seen.add(k);
return true;
});
for (let i = 0; i < rows.length; i += 100) {
const safe = (n: number | null) => (n != null && Number.isFinite(n) && n > 0 ? Math.min(Math.round(n), 2_000_000_000) : null);
const chunk = rows.slice(i, i + 100).map((j) => ({
...j,
title: j.title.replace(/\u0000/g, "").slice(0, 300),
description: j.description.replace(/\u0000/g, ""),
salaryMin: safe(j.salaryMin),
salaryMax: safe(j.salaryMax),
postedAt: isNaN(j.postedAt.getTime()) ? new Date() : j.postedAt,
fetchedAt: new Date(),
}));
await db
.insert(jobs)
.values(chunk)
.onConflictDoUpdate({
target: [jobs.source, jobs.externalId],
set: {
title: sql`excluded.title`,
description: sql`excluded.description`,
tags: sql`excluded.tags`,
salaryMin: sql`excluded.salary_min`,
salaryMax: sql`excluded.salary_max`,
fetchedAt: sql`excluded.fetched_at`,
},
});
}
return rows.length;
}
async function countSource(key: string) {
const r = await db.execute(sql`select count(*)::int as c from jobs where source = ${key}`);
return (r.rows[0] as { c: number }).c;
}
async function run(): Promise<SyncSummary> {
const results = await Promise.all(
SOURCES.map(async (s) => {
const t0 = Date.now();
try {
const before = await countSource(s.key);
const list = await s.fetch();
const n = await upsert(list);
const after = await countSource(s.key);
const inserted = Math.max(0, after - before);
await db.insert(syncRuns).values({ source: s.key, fetched: n, inserted, status: "ok", durationMs: Date.now() - t0 });
return { key: s.key, fetched: n, inserted };
} catch (e) {
const cause = (e as { cause?: { message?: string } })?.cause?.message;
const error = cause || (e instanceof Error ? e.message : String(e));
await db.insert(syncRuns).values({ source: s.key, status: "error", error: error.slice(0, 200), durationMs: Date.now() - t0 });
return { key: s.key, fetched: 0, inserted: 0, error };
}
}),
);
await db.delete(jobs).where(lt(jobs.postedAt, new Date(Date.now() - 45 * 86400_000)));
return {
fetched: results.reduce((a, r) => a + r.fetched, 0),
inserted: results.reduce((a, r) => a + r.inserted, 0),
sources: results,
};
}
export function syncAllSources(): Promise<SyncSummary> {
if (!g.__hfSync) {
g.__hfSync = run().finally(() => {
g.__hfSync = null;
});
}
return g.__hfSync;
}
export async function lastSyncAt(): Promise<Date | null> {
const r = await db.select({ at: syncRuns.startedAt }).from(syncRuns).orderBy(desc(syncRuns.startedAt)).limit(1);
return r[0]?.at ?? null;
}
export async function jobCount() {
const r = await db.execute(sql`select count(*)::int as c from jobs`);
return (r.rows[0] as { c: number }).c;
}