Download src/lib/sync.ts from lljz66/api: direct link, hf CLI and curl.
- Browser
- Download file 3.7 kB
-
https://huggingface.co/spaces/lljz66/api/resolve/main/src/lib/sync.ts
- Command line
-
hf download hf://spaces/lljz66/api/src/lib/sync.ts
-
curl -L -o sync.ts https://huggingface.co/spaces/lljz66/api/resolve/main/src/lib/sync.ts
3.7 kB
| import { desc, lt, sql } from "drizzle-orm"; | |
| import { db } from "@/db"; | |
| import { jobs, syncRuns } from "@/db/schema"; | |
| import { SOURCES, type NJob } from "./sources"; | |
| const g = globalThis as typeof globalThis & { __hfSync?: Promise<SyncSummary> | null }; | |
| export type SyncSummary = { fetched: number; inserted: number; sources: { key: string; fetched: number; inserted: number; error?: string }[] }; | |
| async function upsert(list: NJob[]) { | |
| const seen = new Set<string>(); | |
| const cutoff = Date.now() - 44 * 86400_000; | |
| const rows = list.filter((j) => { | |
| const k = `${j.source}:${j.externalId}`; | |
| if (seen.has(k) || !j.title || !j.url) return false; | |
| if (!isNaN(j.postedAt.getTime()) && j.postedAt.getTime() < cutoff) return false; | |
| seen.add(k); | |
| return true; | |
| }); | |
| for (let i = 0; i < rows.length; i += 100) { | |
| const safe = (n: number | null) => (n != null && Number.isFinite(n) && n > 0 ? Math.min(Math.round(n), 2_000_000_000) : null); | |
| const chunk = rows.slice(i, i + 100).map((j) => ({ | |
| ...j, | |
| title: j.title.replace(/\u0000/g, "").slice(0, 300), | |
| description: j.description.replace(/\u0000/g, ""), | |
| salaryMin: safe(j.salaryMin), | |
| salaryMax: safe(j.salaryMax), | |
| postedAt: isNaN(j.postedAt.getTime()) ? new Date() : j.postedAt, | |
| fetchedAt: new Date(), | |
| })); | |
| await db | |
| .insert(jobs) | |
| .values(chunk) | |
| .onConflictDoUpdate({ | |
| target: [jobs.source, jobs.externalId], | |
| set: { | |
| title: sql`excluded.title`, | |
| description: sql`excluded.description`, | |
| tags: sql`excluded.tags`, | |
| salaryMin: sql`excluded.salary_min`, | |
| salaryMax: sql`excluded.salary_max`, | |
| fetchedAt: sql`excluded.fetched_at`, | |
| }, | |
| }); | |
| } | |
| return rows.length; | |
| } | |
| async function countSource(key: string) { | |
| const r = await db.execute(sql`select count(*)::int as c from jobs where source = ${key}`); | |
| return (r.rows[0] as { c: number }).c; | |
| } | |
| async function run(): Promise<SyncSummary> { | |
| const results = await Promise.all( | |
| SOURCES.map(async (s) => { | |
| const t0 = Date.now(); | |
| try { | |
| const before = await countSource(s.key); | |
| const list = await s.fetch(); | |
| const n = await upsert(list); | |
| const after = await countSource(s.key); | |
| const inserted = Math.max(0, after - before); | |
| await db.insert(syncRuns).values({ source: s.key, fetched: n, inserted, status: "ok", durationMs: Date.now() - t0 }); | |
| return { key: s.key, fetched: n, inserted }; | |
| } catch (e) { | |
| const cause = (e as { cause?: { message?: string } })?.cause?.message; | |
| const error = cause || (e instanceof Error ? e.message : String(e)); | |
| await db.insert(syncRuns).values({ source: s.key, status: "error", error: error.slice(0, 200), durationMs: Date.now() - t0 }); | |
| return { key: s.key, fetched: 0, inserted: 0, error }; | |
| } | |
| }), | |
| ); | |
| await db.delete(jobs).where(lt(jobs.postedAt, new Date(Date.now() - 45 * 86400_000))); | |
| return { | |
| fetched: results.reduce((a, r) => a + r.fetched, 0), | |
| inserted: results.reduce((a, r) => a + r.inserted, 0), | |
| sources: results, | |
| }; | |
| } | |
| export function syncAllSources(): Promise<SyncSummary> { | |
| if (!g.__hfSync) { | |
| g.__hfSync = run().finally(() => { | |
| g.__hfSync = null; | |
| }); | |
| } | |
| return g.__hfSync; | |
| } | |
| export async function lastSyncAt(): Promise<Date | null> { | |
| const r = await db.select({ at: syncRuns.startedAt }).from(syncRuns).orderBy(desc(syncRuns.startedAt)).limit(1); | |
| return r[0]?.at ?? null; | |
| } | |
| export async function jobCount() { | |
| const r = await db.execute(sql`select count(*)::int as c from jobs`); | |
| return (r.rows[0] as { c: number }).c; | |
| } |