Download web/src/connectors/materializationJobs.ts from fsanyoto/loopable: direct link, hf CLI and curl.
- Browser
- Download file 5.66 kB
-
https://huggingface.co/spaces/fsanyoto/loopable/resolve/9291c0100f4e7b8da2efcbbf1ebdea72ee1663e7/web/src/connectors/materializationJobs.ts
- Command line
-
hf download hf://spaces/fsanyoto/loopable@9291c0100f4e7b8da2efcbbf1ebdea72ee1663e7/web/src/connectors/materializationJobs.ts
-
curl -L -o materializationJobs.ts https://huggingface.co/spaces/fsanyoto/loopable/resolve/9291c0100f4e7b8da2efcbbf1ebdea72ee1663e7/web/src/connectors/materializationJobs.ts
5.66 kB
| import { API_V1, CREDENTIALS, WORKSPACE_STALE_EVENT, signal, checkTenant } from "../apiContract"; | |
| export interface MaterializationCapability { | |
| provider: string; | |
| statusPath: string; | |
| retryPath: string; | |
| } | |
| export interface MaterializationJob { | |
| id: string; | |
| provider: string; | |
| connectionId: string; | |
| databaseKey: string; | |
| state: "configured" | "catalogue-ready" | "queued" | "syncing" | "ready" | "failed"; | |
| attempts: number; | |
| maxAttempts: number; | |
| retryable: boolean; | |
| error: string; | |
| workspaceChanged: boolean; | |
| progress?: { rows: number; total: number | null }; | |
| stall?: { cause: string; at: number }; | |
| } | |
| export function parseMaterializationJobs(raw: unknown, provider: string): MaterializationJob[] { | |
| if (!Array.isArray(raw)) return []; | |
| return raw.flatMap((value) => { | |
| if (!value || typeof value !== "object") return []; | |
| const row = value as Record<string, unknown>; | |
| if (row.provider !== provider || typeof row.id !== "string" || | |
| typeof row.databaseKey !== "string" || typeof row.connectionId !== "string" || | |
| !["configured", "catalogue-ready", "queued", "syncing", "ready", "failed"].includes(String(row.state))) return []; | |
| let progress: MaterializationJob["progress"]; | |
| let stall: MaterializationJob["stall"]; | |
| if (row.progress && typeof row.progress === "object" && !Array.isArray(row.progress)) { | |
| const value = row.progress as Record<string, unknown>; | |
| if (typeof value.rows === "number" && Number.isSafeInteger(value.rows) && value.rows >= 0 && | |
| (value.total === null || (typeof value.total === "number" && | |
| Number.isSafeInteger(value.total) && value.total >= 0))) { | |
| progress = { rows: value.rows, total: value.total }; | |
| } | |
| } | |
| if (row.stall && typeof row.stall === "object" && !Array.isArray(row.stall)) { | |
| const value = row.stall as Record<string, unknown>; | |
| if (typeof value.cause === "string" && value.cause.trim() && | |
| typeof value.at === "number" && Number.isFinite(value.at) && value.at >= 0) { | |
| stall = { cause: value.cause, at: value.at }; | |
| } | |
| } | |
| return [{ | |
| id: row.id, provider, connectionId: row.connectionId, databaseKey: row.databaseKey, | |
| state: row.state as MaterializationJob["state"], | |
| attempts: typeof row.attempts === "number" ? row.attempts : 0, | |
| maxAttempts: typeof row.maxAttempts === "number" ? row.maxAttempts : 0, | |
| retryable: row.retryable === true, error: typeof row.error === "string" ? row.error : "", | |
| workspaceChanged: row.workspaceChanged === true, | |
| ...(progress ? { progress } : {}), | |
| ...(stall ? { stall } : {}), | |
| }]; | |
| }); | |
| } | |
| type Listener = (jobs: MaterializationJob[], error: string) => void; | |
| type Watch = { listeners: Set<Listener>; timer?: ReturnType<typeof setTimeout>; | |
| running: boolean; again: boolean; failures: number; announced: Set<string>; | |
| published: Map<string, number> }; | |
| const watches = new Map<string, Watch>(); | |
| /** Observation outlives the modal; execution belongs to the durable server worker. */ | |
| export function watchMaterialization(capability: MaterializationCapability, listener?: Listener): () => void { | |
| const key = capability.statusPath; | |
| let watch = watches.get(key); | |
| if (!watch) { | |
| watch = { listeners: new Set(), running: false, again: false, failures: 0, | |
| announced: new Set(), published: new Map() }; | |
| watches.set(key, watch); | |
| } | |
| const current = watch; | |
| if (listener) current.listeners.add(listener); | |
| async function poll() { | |
| if (current.running) { current.again = true; return; } | |
| current.running = true; | |
| current.again = false; | |
| let active = false; | |
| let refused = false; | |
| try { | |
| const response = await fetch(`${API_V1}${capability.statusPath}`, { credentials: CREDENTIALS }); | |
| if (checkTenant(response) || response.status === 401 || response.status === 403) { | |
| refused = true; | |
| throw new Error("Your session changed. Reopen this connector."); | |
| } | |
| if (!response.ok) throw new Error("Sync status is unavailable. Reopen this connector to check again."); | |
| const body = await response.json() as { jobs?: unknown }; | |
| const jobs = parseMaterializationJobs(body.jobs, capability.provider); | |
| current.failures = 0; | |
| active = jobs.some((job) => job.state === "queued" || job.state === "syncing"); | |
| for (const job of jobs) { | |
| if (job.state === "syncing" && job.progress && job.progress.rows > 0 && | |
| current.published.get(job.id) !== job.progress.rows) { | |
| current.published.set(job.id, job.progress.rows); | |
| signal(WORKSPACE_STALE_EVENT); | |
| } | |
| if (job.state === "ready" && job.workspaceChanged && !current.announced.has(job.id)) { | |
| current.announced.add(job.id); | |
| signal(WORKSPACE_STALE_EVENT); | |
| } | |
| if (job.state !== "syncing") current.published.delete(job.id); | |
| } | |
| current.listeners.forEach((notify) => notify(jobs, "")); | |
| } catch (error) { | |
| current.failures += 1; | |
| active = !refused && current.failures < 5; | |
| current.listeners.forEach((notify) => notify([], error instanceof Error ? error.message : "Sync status is unavailable.")); | |
| } finally { | |
| current.running = false; | |
| if (!refused && (current.again || active || (current.listeners.size > 0 && current.failures < 5))) { | |
| current.timer = setTimeout(() => void poll(), current.again ? 0 : active ? 1500 : 10000); | |
| } else if (!current.listeners.size || refused) { | |
| watches.delete(key); | |
| } | |
| } | |
| } | |
| clearTimeout(current.timer); | |
| void poll(); | |
| return () => { if (listener) current.listeners.delete(listener); }; | |
| } | |