lljz66 commited on
Commit
05950f5
·
verified ·
1 Parent(s): cbf83bb

Put src/lib/sync.ts

Browse files
Files changed (1) hide show
  1. src/lib/sync.ts +99 -0
src/lib/sync.ts ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { desc, lt, sql } from "drizzle-orm";
2
+ import { db } from "@/db";
3
+ import { jobs, syncRuns } from "@/db/schema";
4
+ import { SOURCES, type NJob } from "./sources";
5
+
6
+ const g = globalThis as typeof globalThis & { __hfSync?: Promise<SyncSummary> | null };
7
+
8
+ export type SyncSummary = { fetched: number; inserted: number; sources: { key: string; fetched: number; inserted: number; error?: string }[] };
9
+
10
+ async function upsert(list: NJob[]) {
11
+ const seen = new Set<string>();
12
+ const cutoff = Date.now() - 44 * 86400_000;
13
+ const rows = list.filter((j) => {
14
+ const k = `${j.source}:${j.externalId}`;
15
+ if (seen.has(k) || !j.title || !j.url) return false;
16
+ if (!isNaN(j.postedAt.getTime()) && j.postedAt.getTime() < cutoff) return false;
17
+ seen.add(k);
18
+ return true;
19
+ });
20
+ for (let i = 0; i < rows.length; i += 100) {
21
+ const safe = (n: number | null) => (n != null && Number.isFinite(n) && n > 0 ? Math.min(Math.round(n), 2_000_000_000) : null);
22
+ const chunk = rows.slice(i, i + 100).map((j) => ({
23
+ ...j,
24
+ title: j.title.replace(/\u0000/g, "").slice(0, 300),
25
+ description: j.description.replace(/\u0000/g, ""),
26
+ salaryMin: safe(j.salaryMin),
27
+ salaryMax: safe(j.salaryMax),
28
+ postedAt: isNaN(j.postedAt.getTime()) ? new Date() : j.postedAt,
29
+ fetchedAt: new Date(),
30
+ }));
31
+ await db
32
+ .insert(jobs)
33
+ .values(chunk)
34
+ .onConflictDoUpdate({
35
+ target: [jobs.source, jobs.externalId],
36
+ set: {
37
+ title: sql`excluded.title`,
38
+ description: sql`excluded.description`,
39
+ tags: sql`excluded.tags`,
40
+ salaryMin: sql`excluded.salary_min`,
41
+ salaryMax: sql`excluded.salary_max`,
42
+ fetchedAt: sql`excluded.fetched_at`,
43
+ },
44
+ });
45
+ }
46
+ return rows.length;
47
+ }
48
+
49
+ async function countSource(key: string) {
50
+ const r = await db.execute(sql`select count(*)::int as c from jobs where source = ${key}`);
51
+ return (r.rows[0] as { c: number }).c;
52
+ }
53
+
54
+ async function run(): Promise<SyncSummary> {
55
+ const results = await Promise.all(
56
+ SOURCES.map(async (s) => {
57
+ const t0 = Date.now();
58
+ try {
59
+ const before = await countSource(s.key);
60
+ const list = await s.fetch();
61
+ const n = await upsert(list);
62
+ const after = await countSource(s.key);
63
+ const inserted = Math.max(0, after - before);
64
+ await db.insert(syncRuns).values({ source: s.key, fetched: n, inserted, status: "ok", durationMs: Date.now() - t0 });
65
+ return { key: s.key, fetched: n, inserted };
66
+ } catch (e) {
67
+ const cause = (e as { cause?: { message?: string } })?.cause?.message;
68
+ const error = cause || (e instanceof Error ? e.message : String(e));
69
+ await db.insert(syncRuns).values({ source: s.key, status: "error", error: error.slice(0, 200), durationMs: Date.now() - t0 });
70
+ return { key: s.key, fetched: 0, inserted: 0, error };
71
+ }
72
+ }),
73
+ );
74
+ await db.delete(jobs).where(lt(jobs.postedAt, new Date(Date.now() - 45 * 86400_000)));
75
+ return {
76
+ fetched: results.reduce((a, r) => a + r.fetched, 0),
77
+ inserted: results.reduce((a, r) => a + r.inserted, 0),
78
+ sources: results,
79
+ };
80
+ }
81
+
82
+ export function syncAllSources(): Promise<SyncSummary> {
83
+ if (!g.__hfSync) {
84
+ g.__hfSync = run().finally(() => {
85
+ g.__hfSync = null;
86
+ });
87
+ }
88
+ return g.__hfSync;
89
+ }
90
+
91
+ export async function lastSyncAt(): Promise<Date | null> {
92
+ const r = await db.select({ at: syncRuns.startedAt }).from(syncRuns).orderBy(desc(syncRuns.startedAt)).limit(1);
93
+ return r[0]?.at ?? null;
94
+ }
95
+
96
+ export async function jobCount() {
97
+ const r = await db.execute(sql`select count(*)::int as c from jobs`);
98
+ return (r.rows[0] as { c: number }).c;
99
+ }