File size: 3,696 Bytes
05950f5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
import { desc, lt, sql } from "drizzle-orm";
import { db } from "@/db";
import { jobs, syncRuns } from "@/db/schema";
import { SOURCES, type NJob } from "./sources";

const g = globalThis as typeof globalThis & { __hfSync?: Promise<SyncSummary> | null };

export type SyncSummary = { fetched: number; inserted: number; sources: { key: string; fetched: number; inserted: number; error?: string }[] };

async function upsert(list: NJob[]) {
  const seen = new Set<string>();
  const cutoff = Date.now() - 44 * 86400_000;
  const rows = list.filter((j) => {
    const k = `${j.source}:${j.externalId}`;
    if (seen.has(k) || !j.title || !j.url) return false;
    if (!isNaN(j.postedAt.getTime()) && j.postedAt.getTime() < cutoff) return false;
    seen.add(k);
    return true;
  });
  for (let i = 0; i < rows.length; i += 100) {
    const safe = (n: number | null) => (n != null && Number.isFinite(n) && n > 0 ? Math.min(Math.round(n), 2_000_000_000) : null);
    const chunk = rows.slice(i, i + 100).map((j) => ({
      ...j,
      title: j.title.replace(/\u0000/g, "").slice(0, 300),
      description: j.description.replace(/\u0000/g, ""),
      salaryMin: safe(j.salaryMin),
      salaryMax: safe(j.salaryMax),
      postedAt: isNaN(j.postedAt.getTime()) ? new Date() : j.postedAt,
      fetchedAt: new Date(),
    }));
    await db
      .insert(jobs)
      .values(chunk)
      .onConflictDoUpdate({
        target: [jobs.source, jobs.externalId],
        set: {
          title: sql`excluded.title`,
          description: sql`excluded.description`,
          tags: sql`excluded.tags`,
          salaryMin: sql`excluded.salary_min`,
          salaryMax: sql`excluded.salary_max`,
          fetchedAt: sql`excluded.fetched_at`,
        },
      });
  }
  return rows.length;
}

async function countSource(key: string) {
  const r = await db.execute(sql`select count(*)::int as c from jobs where source = ${key}`);
  return (r.rows[0] as { c: number }).c;
}

async function run(): Promise<SyncSummary> {
  const results = await Promise.all(
    SOURCES.map(async (s) => {
      const t0 = Date.now();
      try {
        const before = await countSource(s.key);
        const list = await s.fetch();
        const n = await upsert(list);
        const after = await countSource(s.key);
        const inserted = Math.max(0, after - before);
        await db.insert(syncRuns).values({ source: s.key, fetched: n, inserted, status: "ok", durationMs: Date.now() - t0 });
        return { key: s.key, fetched: n, inserted };
      } catch (e) {
        const cause = (e as { cause?: { message?: string } })?.cause?.message;
        const error = cause || (e instanceof Error ? e.message : String(e));
        await db.insert(syncRuns).values({ source: s.key, status: "error", error: error.slice(0, 200), durationMs: Date.now() - t0 });
        return { key: s.key, fetched: 0, inserted: 0, error };
      }
    }),
  );
  await db.delete(jobs).where(lt(jobs.postedAt, new Date(Date.now() - 45 * 86400_000)));
  return {
    fetched: results.reduce((a, r) => a + r.fetched, 0),
    inserted: results.reduce((a, r) => a + r.inserted, 0),
    sources: results,
  };
}

export function syncAllSources(): Promise<SyncSummary> {
  if (!g.__hfSync) {
    g.__hfSync = run().finally(() => {
      g.__hfSync = null;
    });
  }
  return g.__hfSync;
}

export async function lastSyncAt(): Promise<Date | null> {
  const r = await db.select({ at: syncRuns.startedAt }).from(syncRuns).orderBy(desc(syncRuns.startedAt)).limit(1);
  return r[0]?.at ?? null;
}

export async function jobCount() {
  const r = await db.execute(sql`select count(*)::int as c from jobs`);
  return (r.rows[0] as { c: number }).c;
}