loopable / web /src /connectors /materializationJobs.ts
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
bdac669 verified
Raw History Blame
5.66 kB
import { API_V1, CREDENTIALS, WORKSPACE_STALE_EVENT, signal, checkTenant } from "../apiContract";
export interface MaterializationCapability {
provider: string;
statusPath: string;
retryPath: string;
}
export interface MaterializationJob {
id: string;
provider: string;
connectionId: string;
databaseKey: string;
state: "configured" | "catalogue-ready" | "queued" | "syncing" | "ready" | "failed";
attempts: number;
maxAttempts: number;
retryable: boolean;
error: string;
workspaceChanged: boolean;
progress?: { rows: number; total: number | null };
stall?: { cause: string; at: number };
}
export function parseMaterializationJobs(raw: unknown, provider: string): MaterializationJob[] {
if (!Array.isArray(raw)) return [];
return raw.flatMap((value) => {
if (!value || typeof value !== "object") return [];
const row = value as Record<string, unknown>;
if (row.provider !== provider || typeof row.id !== "string" ||
typeof row.databaseKey !== "string" || typeof row.connectionId !== "string" ||
!["configured", "catalogue-ready", "queued", "syncing", "ready", "failed"].includes(String(row.state))) return [];
let progress: MaterializationJob["progress"];
let stall: MaterializationJob["stall"];
if (row.progress && typeof row.progress === "object" && !Array.isArray(row.progress)) {
const value = row.progress as Record<string, unknown>;
if (typeof value.rows === "number" && Number.isSafeInteger(value.rows) && value.rows >= 0 &&
(value.total === null || (typeof value.total === "number" &&
Number.isSafeInteger(value.total) && value.total >= 0))) {
progress = { rows: value.rows, total: value.total };
}
}
if (row.stall && typeof row.stall === "object" && !Array.isArray(row.stall)) {
const value = row.stall as Record<string, unknown>;
if (typeof value.cause === "string" && value.cause.trim() &&
typeof value.at === "number" && Number.isFinite(value.at) && value.at >= 0) {
stall = { cause: value.cause, at: value.at };
}
}
return [{
id: row.id, provider, connectionId: row.connectionId, databaseKey: row.databaseKey,
state: row.state as MaterializationJob["state"],
attempts: typeof row.attempts === "number" ? row.attempts : 0,
maxAttempts: typeof row.maxAttempts === "number" ? row.maxAttempts : 0,
retryable: row.retryable === true, error: typeof row.error === "string" ? row.error : "",
workspaceChanged: row.workspaceChanged === true,
...(progress ? { progress } : {}),
...(stall ? { stall } : {}),
}];
});
}
type Listener = (jobs: MaterializationJob[], error: string) => void;
type Watch = { listeners: Set<Listener>; timer?: ReturnType<typeof setTimeout>;
running: boolean; again: boolean; failures: number; announced: Set<string>;
published: Map<string, number> };
const watches = new Map<string, Watch>();
/** Observation outlives the modal; execution belongs to the durable server worker. */
export function watchMaterialization(capability: MaterializationCapability, listener?: Listener): () => void {
const key = capability.statusPath;
let watch = watches.get(key);
if (!watch) {
watch = { listeners: new Set(), running: false, again: false, failures: 0,
announced: new Set(), published: new Map() };
watches.set(key, watch);
}
const current = watch;
if (listener) current.listeners.add(listener);
async function poll() {
if (current.running) { current.again = true; return; }
current.running = true;
current.again = false;
let active = false;
let refused = false;
try {
const response = await fetch(`${API_V1}${capability.statusPath}`, { credentials: CREDENTIALS });
if (checkTenant(response) || response.status === 401 || response.status === 403) {
refused = true;
throw new Error("Your session changed. Reopen this connector.");
}
if (!response.ok) throw new Error("Sync status is unavailable. Reopen this connector to check again.");
const body = await response.json() as { jobs?: unknown };
const jobs = parseMaterializationJobs(body.jobs, capability.provider);
current.failures = 0;
active = jobs.some((job) => job.state === "queued" || job.state === "syncing");
for (const job of jobs) {
if (job.state === "syncing" && job.progress && job.progress.rows > 0 &&
current.published.get(job.id) !== job.progress.rows) {
current.published.set(job.id, job.progress.rows);
signal(WORKSPACE_STALE_EVENT);
}
if (job.state === "ready" && job.workspaceChanged && !current.announced.has(job.id)) {
current.announced.add(job.id);
signal(WORKSPACE_STALE_EVENT);
}
if (job.state !== "syncing") current.published.delete(job.id);
}
current.listeners.forEach((notify) => notify(jobs, ""));
} catch (error) {
current.failures += 1;
active = !refused && current.failures < 5;
current.listeners.forEach((notify) => notify([], error instanceof Error ? error.message : "Sync status is unavailable."));
} finally {
current.running = false;
if (!refused && (current.again || active || (current.listeners.size > 0 && current.failures < 5))) {
current.timer = setTimeout(() => void poll(), current.again ? 0 : active ? 1500 : 10000);
} else if (!current.listeners.size || refused) {
watches.delete(key);
}
}
}
clearTimeout(current.timer);
void poll();
return () => { if (listener) current.listeners.delete(listener); };
}