Put src/lib/sync.ts
Browse files- src/lib/sync.ts +99 -0
src/lib/sync.ts
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { desc, lt, sql } from "drizzle-orm";
|
| 2 |
+
import { db } from "@/db";
|
| 3 |
+
import { jobs, syncRuns } from "@/db/schema";
|
| 4 |
+
import { SOURCES, type NJob } from "./sources";
|
| 5 |
+
|
| 6 |
+
const g = globalThis as typeof globalThis & { __hfSync?: Promise<SyncSummary> | null };
|
| 7 |
+
|
| 8 |
+
export type SyncSummary = { fetched: number; inserted: number; sources: { key: string; fetched: number; inserted: number; error?: string }[] };
|
| 9 |
+
|
| 10 |
+
async function upsert(list: NJob[]) {
|
| 11 |
+
const seen = new Set<string>();
|
| 12 |
+
const cutoff = Date.now() - 44 * 86400_000;
|
| 13 |
+
const rows = list.filter((j) => {
|
| 14 |
+
const k = `${j.source}:${j.externalId}`;
|
| 15 |
+
if (seen.has(k) || !j.title || !j.url) return false;
|
| 16 |
+
if (!isNaN(j.postedAt.getTime()) && j.postedAt.getTime() < cutoff) return false;
|
| 17 |
+
seen.add(k);
|
| 18 |
+
return true;
|
| 19 |
+
});
|
| 20 |
+
for (let i = 0; i < rows.length; i += 100) {
|
| 21 |
+
const safe = (n: number | null) => (n != null && Number.isFinite(n) && n > 0 ? Math.min(Math.round(n), 2_000_000_000) : null);
|
| 22 |
+
const chunk = rows.slice(i, i + 100).map((j) => ({
|
| 23 |
+
...j,
|
| 24 |
+
title: j.title.replace(/\u0000/g, "").slice(0, 300),
|
| 25 |
+
description: j.description.replace(/\u0000/g, ""),
|
| 26 |
+
salaryMin: safe(j.salaryMin),
|
| 27 |
+
salaryMax: safe(j.salaryMax),
|
| 28 |
+
postedAt: isNaN(j.postedAt.getTime()) ? new Date() : j.postedAt,
|
| 29 |
+
fetchedAt: new Date(),
|
| 30 |
+
}));
|
| 31 |
+
await db
|
| 32 |
+
.insert(jobs)
|
| 33 |
+
.values(chunk)
|
| 34 |
+
.onConflictDoUpdate({
|
| 35 |
+
target: [jobs.source, jobs.externalId],
|
| 36 |
+
set: {
|
| 37 |
+
title: sql`excluded.title`,
|
| 38 |
+
description: sql`excluded.description`,
|
| 39 |
+
tags: sql`excluded.tags`,
|
| 40 |
+
salaryMin: sql`excluded.salary_min`,
|
| 41 |
+
salaryMax: sql`excluded.salary_max`,
|
| 42 |
+
fetchedAt: sql`excluded.fetched_at`,
|
| 43 |
+
},
|
| 44 |
+
});
|
| 45 |
+
}
|
| 46 |
+
return rows.length;
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
async function countSource(key: string) {
|
| 50 |
+
const r = await db.execute(sql`select count(*)::int as c from jobs where source = ${key}`);
|
| 51 |
+
return (r.rows[0] as { c: number }).c;
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
async function run(): Promise<SyncSummary> {
|
| 55 |
+
const results = await Promise.all(
|
| 56 |
+
SOURCES.map(async (s) => {
|
| 57 |
+
const t0 = Date.now();
|
| 58 |
+
try {
|
| 59 |
+
const before = await countSource(s.key);
|
| 60 |
+
const list = await s.fetch();
|
| 61 |
+
const n = await upsert(list);
|
| 62 |
+
const after = await countSource(s.key);
|
| 63 |
+
const inserted = Math.max(0, after - before);
|
| 64 |
+
await db.insert(syncRuns).values({ source: s.key, fetched: n, inserted, status: "ok", durationMs: Date.now() - t0 });
|
| 65 |
+
return { key: s.key, fetched: n, inserted };
|
| 66 |
+
} catch (e) {
|
| 67 |
+
const cause = (e as { cause?: { message?: string } })?.cause?.message;
|
| 68 |
+
const error = cause || (e instanceof Error ? e.message : String(e));
|
| 69 |
+
await db.insert(syncRuns).values({ source: s.key, status: "error", error: error.slice(0, 200), durationMs: Date.now() - t0 });
|
| 70 |
+
return { key: s.key, fetched: 0, inserted: 0, error };
|
| 71 |
+
}
|
| 72 |
+
}),
|
| 73 |
+
);
|
| 74 |
+
await db.delete(jobs).where(lt(jobs.postedAt, new Date(Date.now() - 45 * 86400_000)));
|
| 75 |
+
return {
|
| 76 |
+
fetched: results.reduce((a, r) => a + r.fetched, 0),
|
| 77 |
+
inserted: results.reduce((a, r) => a + r.inserted, 0),
|
| 78 |
+
sources: results,
|
| 79 |
+
};
|
| 80 |
+
}
|
| 81 |
+
|
| 82 |
+
export function syncAllSources(): Promise<SyncSummary> {
|
| 83 |
+
if (!g.__hfSync) {
|
| 84 |
+
g.__hfSync = run().finally(() => {
|
| 85 |
+
g.__hfSync = null;
|
| 86 |
+
});
|
| 87 |
+
}
|
| 88 |
+
return g.__hfSync;
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
export async function lastSyncAt(): Promise<Date | null> {
|
| 92 |
+
const r = await db.select({ at: syncRuns.startedAt }).from(syncRuns).orderBy(desc(syncRuns.startedAt)).limit(1);
|
| 93 |
+
return r[0]?.at ?? null;
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
export async function jobCount() {
|
| 97 |
+
const r = await db.execute(sql`select count(*)::int as c from jobs`);
|
| 98 |
+
return (r.rows[0] as { c: number }).c;
|
| 99 |
+
}
|