indra / engine.js
ExplodingCB2's picture
Show a progress bar while the forecast loads
538befb verified
Raw History Blame Contribute Delete
27.7 kB
"use strict";
// Everything between public weather data and a forecast, on the visitor's own device: no server involved.
// The steps and their order match the Python that trained and scored the model (build.py, dataset.py,
// fastpath.py, timestral's scaler.py). scripts/browser_parity.py checks the two give the same forecast.
const Engine = (() => {
const MODEL_URL = window.INDRA_MODEL || "https://huggingface.co/Sculptor-AI/Indra-3M/resolve/main/indra-3m.onnx";
const ORT_BASE = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.30.0/dist/";
const IEM = "https://mesonet.agron.iastate.edu/cgi-bin/request";
const OBS_VARS = ["tmpf", "dwpf", "relh", "drct", "sknt", "gust", "mslp", "alti", "p01i", "vsby", "skyc1", "skyl1", "skyc2", "skyl2", "skyc3", "skyl3", "wxcodes"];
const TARGETS = ["t", "td", "wspd", "u", "v", "mslp"], PAST = ["gust", "sky", "precip", "vis"];
const SOURCES = [
{ name: "NBS", delay: 2, vars: ["t", "td", "wspd", "u", "v", "sky", "gust"] },
{ name: "GFS", delay: 5, vars: ["t", "td", "wspd", "u", "v"] },
{ name: "NAM", delay: 4, vars: ["t", "td", "wspd", "u", "v"] },
];
const KT = 0.514444, HOUR = 36e5, CONTEXT = 2160, H = 72, DAYS = 96, MAX_LEAD = 84, FALLBACK = 4;
const PS = 32, PATCHES_OUT = 3, Q = 21, MEDIAN = 10, MIN_SCALE = 1e-6, EPS = 1.1920928955078125e-7;
const MODEL_BYTES = 12396384, WASM_BYTES = 14239897;
const LEVELS = { "0.05": 1, "0.1": 2, "0.25": 5, "0.5": 10, "0.75": 15, "0.9": 18, "0.95": 19 };
const SKY = { CLR: 0, SKC: 0, NSC: 0, NCD: 0, FEW: 19, SCT: 44, BKN: 75, OVC: 100, VV: 100 };
const f = Math.fround;
// ---------------------------------------------------------------- reading the archive's text
const num = (s) => (s === undefined || s === "" ? NaN : Number(s));
const when = (s) => {
const m = /^(\d{4})-(\d\d)-(\d\d)[ T](\d\d):(\d\d)(?::(\d\d))?/.exec(s);
return m ? Date.UTC(+m[1], +m[2] - 1, +m[3], +m[4], +m[5], +(m[6] || 0)) : NaN;
};
function csv(text) {
const lines = text.split(/\r?\n/).filter((l) => l && l[0] !== "#");
if (!lines.length) return [];
const head = lines[0].split(",").map((x) => x.trim());
return lines.slice(1).map((l) => {
const c = l.split(","), o = {};
head.forEach((k, i) => { o[k] = c[i] === undefined ? "" : c[i].trim(); });
return o;
});
}
const windUV = (speed, dir) => {
if (speed === 0) return [0, 0];
const r = dir * Math.PI / 180;
return [-speed * Math.sin(r), -speed * Math.cos(r)];
};
const f2c = (x) => (x - 32) * 5 / 9;
function buildObs(text) {
const best = new Map();
for (const r of csv(text)) {
const t = when(r.valid);
if (!Number.isFinite(t)) continue;
const stamp = Math.floor((t + 18e5) / HOUR) * HOUR, off = (t - stamp) / 6e4;
if (off < -30 || off > 5) continue;
const dist = Math.abs(off + 6), had = best.get(stamp);
if (!had || dist < had.dist) best.set(stamp, { dist, r });
}
if (!best.size) return null;
const stamps = [...best.keys()];
const start = Math.min(...stamps) / HOUR, stop = Math.max(...stamps) / HOUR, n = stop - start + 1 + H + 1;
const col = () => new Float64Array(n).fill(NaN);
const o = { start, n, t: col(), td: col(), wspd: col(), u: col(), v: col(), mslp: col(), gust: col(), sky: col(), precip: col(), vis: col() };
for (const [stamp, { r }] of best) {
const i = stamp / HOUR - start, spd = num(r.sknt) * KT;
let dir = num(r.drct);
if (!(dir >= 0 && dir <= 360)) dir = NaN;
const [u, v] = windUV(spd, dir);
let t = f2c(num(r.tmpf)), td = f2c(num(r.dwpf)), p = num(r.mslp);
if (t < -40 || t > 45) t = NaN;
if (td < -50 || td > 35 || td > t + 0.6) td = NaN;
if (p < 950 || p > 1065) p = NaN;
const g = num(r.gust) * KT;
const layers = [r.skyc1, r.skyc2, r.skyc3].map((x) => SKY[x]).filter((x) => x !== undefined);
o.t[i] = t; o.td[i] = td; o.wspd[i] = spd; o.u[i] = u; o.v[i] = v; o.mslp[i] = p;
o.gust[i] = Number.isNaN(g) ? spd : g;
o.sky[i] = layers.length ? Math.max(...layers) : NaN;
o.precip[i] = num(r.p01i) * 25.4;
o.vis[i] = num(r.vsby);
}
return o;
}
// Each run's 3-hourly values become hourly by straight lines, at most 5 hours past a known value.
function fill(a) {
let last = -1;
for (let i = 0; i < a.length; i++) {
if (Number.isNaN(a[i])) continue;
if (last >= 0 && i - last > 1) {
const step = (a[i] - a[last]) / (i - last);
for (let j = last + 1; j <= Math.min(i - 1, last + 5); j++) a[j] = a[last] + step * (j - last);
}
last = i;
}
}
function buildSource(src, text, axis) {
const rows = csv(text);
const dirs = rows.map((r) => num(r.wdr)).filter((x) => !Number.isNaN(x));
const tens = dirs.length && Math.max(...dirs) <= 36;
const byRun = new Map();
for (const r of rows) {
const run = when(r.runtime) / HOUR, valid = when(r.ftime) / HOUR, t = f2c(num(r.tmp));
if (!Number.isFinite(run) || !Number.isFinite(valid) || Number.isNaN(t)) continue;
if (!byRun.has(run)) byRun.set(run, new Map());
const g = byRun.get(run);
if (g.has(valid)) continue;
const spd = num(r.wsp) * KT, [u, v] = windUV(spd, num(r.wdr) * (tens ? 10 : 1));
g.set(valid, { t, td: f2c(num(r.dpt)), wspd: spd, u, v, sky: num(r.sky), gust: num(r.gst) * KT });
}
const runs = [...byRun.keys()].sort((a, b) => a - b), L = MAX_LEAD + 1, V = src.vars.length;
const kept = [], vals = [];
for (const run of runs) {
const g = byRun.get(run), times = [...g.keys()], lo = Math.min(...times), hi = Math.max(...times);
const cols = src.vars.map((name) => {
const a = new Float64Array(hi - lo + 1).fill(NaN);
for (const [valid, row] of g) a[valid - lo] = row[name];
fill(a);
return a;
});
const m = new Float64Array(L * V).fill(NaN);
let any = false;
for (let valid = lo; valid <= hi; valid++) {
const lead = valid - run;
if (lead < 0 || lead > MAX_LEAD) continue;
any = true;
for (let c = 0; c < V; c++) m[lead * V + c] = cols[c][valid - lo];
}
if (any) { kept.push(run); vals.push(m); }
}
const hist = new Float64Array(axis.n * V).fill(NaN);
kept.forEach((run, k) => {
const avail = run + src.delay;
for (let lead = 0; lead < L; lead++) {
const valid = run + lead, i = valid - axis.start;
if (valid < avail || i < 0 || i >= axis.n || Number.isNaN(vals[k][lead * V])) continue;
for (let c = 0; c < V; c++) hist[i * V + c] = vals[k][lead * V + c];
}
});
return { ...src, V, runs: kept, vals, hist, missing: !kept.length };
}
// The forecast each source had published by the origin: newest run first, older ones fill what it doesn't reach.
function future(s, origin) {
const out = new Float64Array(H * s.V).fill(NaN);
let k = -1;
for (let i = 0; i < s.runs.length; i++) if (s.runs[i] + s.delay <= origin) k = i;
for (let kk = k; kk > Math.max(k - FALLBACK, -1); kk--) {
let gap = false;
for (let t = 0; t < H; t++) {
if (!Number.isNaN(out[t * s.V])) continue;
const lead = origin + 1 + t - s.runs[kk];
if (lead >= 0 && lead <= MAX_LEAD) for (let c = 0; c < s.V; c++) out[t * s.V + c] = s.vals[kk][lead * s.V + c];
if (Number.isNaN(out[t * s.V])) gap = true;
}
if (!gap) break;
}
return { out, run: k >= 0 ? s.runs[k] : null };
}
function calendar(hour, lon) {
const hod = (((hour + lon / 15) % 24) + 24) % 24 / 24;
const d = new Date(hour * HOUR);
const doy = (Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()) - Date.UTC(d.getUTCFullYear(), 0, 1)) / 864e5 + 1;
const a = 2 * Math.PI * hod, b = 2 * Math.PI * (doy / 365.25);
return [Math.sin(a), Math.cos(a), Math.sin(b), Math.cos(b)];
}
// Rows for one forecast: 6 targets, 4 measured-only, then every guidance row and the calendar.
function window_(obs, sources, origin, lon) {
const T = CONTEXT, start = origin - T + 1;
const at = (name, hour) => { const i = hour - obs.start; return i >= 0 && i < obs.n ? f(obs[name][i]) : NaN; };
const context = TARGETS.map((name) => Float32Array.from({ length: T }, (_, t) => at(name, start + t)));
const past = PAST.map((name) => Float32Array.from({ length: T }, (_, t) => at(name, start + t)));
const known = [];
for (const s of sources) {
const { out } = future(s, origin);
for (let c = 0; c < s.V; c++) {
const row = new Float32Array(T + H);
for (let t = 0; t < T; t++) {
const i = start + t - obs.start;
row[t] = i >= 0 && i < obs.n ? s.hist[i * s.V + c] : NaN;
}
for (let t = 0; t < H; t++) row[T + t] = out[t * s.V + c];
known.push(row);
}
}
for (let c = 0; c < 4; c++) known.push(Float32Array.from({ length: T + H }, (_, t) => calendar(start + t, lon)[c]));
return { context, past, known };
}
// ---------------------------------------------------------------- what timestral does before the network
function assemble(w, sign) {
const T = CONTEXT, pad = PS - (T % PS), W = T + pad + PATCHES_OUT * PS, nT = w.context.length, R = nT + w.past.length + w.known.length;
const values = new Float32Array(R * W), seen = new Uint8Array(R * W);
const set = (r, t, v, ok) => { values[r * W + t] = v; seen[r * W + t] = ok ? 1 : 0; };
w.context.forEach((row, r) => {
const first = Number.isNaN(row[0]) ? 0 : sign * row[0];
let held = first;
for (let t = 0; t < T; t++) {
const ok = !Number.isNaN(row[t]);
if (ok) held = sign * row[t];
set(r, pad + t, held, ok);
}
});
w.past.forEach((row, p) => {
for (let t = 0; t < T; t++) {
const ok = !Number.isNaN(row[t]);
set(nT + p, pad + t, ok ? sign * row[t] : 0, ok);
}
});
w.known.forEach((row, k) => {
for (let t = 0; t < T + H; t++) {
const ok = !Number.isNaN(row[t]);
set(nT + w.past.length + k, pad + t, ok ? sign * row[t] : 0, ok);
}
});
return { values, seen, R, W, nT, nP: w.past.length, nK: w.known.length, Tp: T + pad };
}
function scale(a) {
const { values, seen, R, W, nT, nP } = a, n = W / PS, ll = n - PATCHES_OUT;
const scaled = new Float32Array(R * W), loc = new Float64Array(R), asc = new Float64Array(R);
for (let r = 0; r < R; r++) {
const at = r * W;
if (r >= nT + nP) { // known-future rows use the whole row's statistics
let c = 0, sum = 0, binary = true;
for (let t = 0; t < W; t++) if (seen[at + t]) {
c++; sum += values[at + t];
if (values[at + t] !== 0 && values[at + t] !== 1) binary = false;
}
const den = Math.max(c, 1);
let mean = sum / den, v = 0;
for (let t = 0; t < W; t++) if (seen[at + t]) v += (values[at + t] - mean) ** 2;
let sd = Math.max(Math.sqrt(v / den), MIN_SCALE);
const isBinary = binary && c > 0;
if (isBinary) { mean = 0; sd = 1; }
mean = f(mean); sd = f(sd);
for (let t = 0; t < W; t++) {
if (!seen[at + t]) { scaled[at + t] = 0; continue; }
const z = f((values[at + t] - mean) / Math.max(sd, EPS));
scaled[at + t] = isBinary ? z : Math.asinh(z);
}
continue;
}
// everything else uses statistics of what was observed up to the end of each patch
let cw = 0, cx = 0, m2 = 0, prev = 0, pl = 0, ps = 1;
for (let t = 0; t < W; t++) {
const x = values[at + t], w = seen[at + t];
cw += w; cx += w * x;
const den = Math.max(cw, 1), mean = cx / den;
m2 += (x - prev) * (x - mean) * w;
prev = mean;
if ((t + 1) % PS === 0) {
pl = f(mean);
ps = f(Math.max(Math.sqrt(m2 / Math.max(den - 1, 1)), MIN_SCALE));
const j = (t + 1) / PS - 1;
if (j === ll - 1) { loc[r] = pl; asc[r] = ps; }
for (let u = t + 1 - PS; u <= t; u++) {
scaled[at + u] = seen[at + u] ? Math.asinh(f((values[at + u] - pl) / Math.max(ps, EPS))) : 0;
}
}
}
}
return { scaled, loc, asc };
}
function masks(a) {
const { seen, R, W, nT, nP } = a, n = W / PS, ll = n - PATCHES_OUT, h = PATCHES_OUT, S = n + h;
const obsPatch = new Uint8Array(R * n), valid = new Uint8Array(R * n);
for (let r = 0; r < R; r++) {
let started = 0;
for (let j = 0; j < n; j++) {
let any = 0;
for (let t = 0; t < PS; t++) if (seen[r * W + j * PS + t]) { any = 1; break; }
obsPatch[r * n + j] = any;
started = started || any;
valid[r * n + j] = started;
}
}
const isKnown = (r) => r >= nT + nP, isTarget = (r) => r < nT;
const time = new Uint8Array(R * S * S);
for (let r = 0; r < R; r++) {
const base = r * S * S;
for (let q = 0; q < n; q++) for (let k = 0; k < n; k++) {
const ok = valid[r * n + k];
time[base + q * S + k] = q === k || (isKnown(r) ? ok : k <= q && ok) ? 1 : 0;
}
for (let q = 0; q < h; q++) {
for (let k = 0; k < n; k++) {
const ok = valid[r * n + k];
time[base + (n + q) * S + k] = isKnown(r) ? ok : k < ll && ok ? 1 : 0; // known rows read their own guidance ahead
}
for (let k = 0; k <= q; k++) time[base + (n + q) * S + n + k] = 1;
}
}
const variate = new Uint8Array(S * R * R);
for (let s = 0; s < S; s++) for (let q = 0; q < R; q++) for (let k = 0; k < R; k++) {
const slot = s < n ? valid[k * n + s] : 1;
const forbid = (!isTarget(q) && isTarget(k)) || (isKnown(q) && !isKnown(k));
variate[s * R * R + q * R + k] = q === k || (slot && !forbid) ? 1 : 0;
}
let first = 0;
for (let j = 0; j < n; j++) {
let any = false;
for (let r = 0; r < R; r++) if (obsPatch[r * n + j]) { any = true; break; }
if (any) { first = j; break; }
}
const positions = new Float32Array(R * S);
for (let r = 0; r < R; r++) {
for (let j = 0; j < n; j++) positions[r * S + j] = j - first;
for (let j = 0; j < h; j++) positions[r * S + n + j] = ll + j - first;
}
return { time, variate, positions, S };
}
const combine = (wa, ma, sa, wb, mb, sb) => {
const w = wa + wb;
if (w === 0) return [0, 0, 0];
const d = mb - ma;
return [w, ma + d * wb / w, sa + sb + d * d * wa * wb / w];
};
const clampInf = (a, from, len) => {
let hi = -Infinity, lo = Infinity, bad = false;
for (let i = from; i < from + len; i++) {
if (a[i] === Infinity || a[i] === -Infinity) bad = true;
else if (Number.isFinite(a[i])) { hi = Math.max(hi, a[i]); lo = Math.min(lo, a[i]); }
}
if (!bad) return;
for (let i = from; i < from + len; i++) {
if (a[i] === Infinity) a[i] = hi === -Infinity ? 0 : hi;
else if (a[i] === -Infinity) a[i] = lo === Infinity ? 0 : lo;
}
};
// Undo the scaling, then move each forecast patch into the frame it was trained in.
function finish(a, raw, sc) {
const { nT, values, seen, W, Tp } = a, L = PATCHES_OUT * PS, n0 = Tp / PS;
const out = new Float32Array(nT * L * Q);
for (let r = 0; r < nT; r++) {
const loc0 = sc.loc[r], scale0 = sc.asc[r], at = r * L * Q;
for (let i = 0; i < L * Q; i++) out[at + i] = f(f(Math.sinh(raw[at + i])) * scale0 + loc0);
for (let t = 0; t < L; t++) clampInf(out, at + t * Q, Q);
let cnt = 0, mean = 0, m2 = 0;
for (let j = 0; j < n0; j++) {
let c = 0, s = 0;
for (let t = 0; t < PS; t++) { const w = seen[r * W + j * PS + t]; c += w; s += w * values[r * W + j * PS + t]; }
if (!c) continue;
const mj = s / Math.max(c, 1);
let vj = 0;
for (let t = 0; t < PS; t++) { const w = seen[r * W + j * PS + t]; vj += w * (values[r * W + j * PS + t] - mj) ** 2; }
[cnt, mean, m2] = combine(cnt, mean, m2, c, mj, vj);
}
const offset = mean, clip = Math.sqrt(3.4028234663852886e38) / 1e3;
const locs = [], scales = [];
for (let h = 0; h < PATCHES_OUT; h++) {
const v = Math.max(m2 / Math.max(cnt - 1, 1), 0), sd = Math.max(Math.sqrt(v), MIN_SCALE);
locs.push(h ? f(mean) : loc0); scales.push(h ? f(sd) : scale0);
const folded = new Float64Array(PS);
let bm = 0;
for (let t = 0; t < PS; t++) {
const med = (out[at + (h * PS + t) * Q + MEDIAN] - loc0) / scale0;
folded[t] = Math.min(clip, Math.max(-clip, med * sd + mean - offset)) + offset;
bm += folded[t];
}
bm /= PS;
let b2 = 0;
for (let t = 0; t < PS; t++) b2 += (folded[t] - bm) ** 2;
[cnt, mean, m2] = combine(cnt, mean, m2, PS, bm, b2);
}
for (let h = 0; h < PATCHES_OUT; h++) for (let t = 0; t < PS; t++) {
const from = at + (h * PS + t) * Q;
for (let q = 0; q < Q; q++) out[from + q] = f(f((out[from + q] - loc0) / scale0) * scales[h] + locs[h]);
clampInf(out, from, Q);
}
}
return out;
}
// ---------------------------------------------------------------- running the network
let session = null;
async function loadRuntime() {
if (window.ort) return;
await new Promise((res, rej) => {
const s = document.createElement("script");
s.src = `${ORT_BASE}ort.wasm.min.js`; s.onload = res; s.onerror = () => rej(new Error("Couldn't load the model runtime."));
document.head.appendChild(s);
});
ort.env.wasm.wasmPaths = ORT_BASE;
ort.env.wasm.numThreads = self.crossOriginIsolated ? Math.min(4, navigator.hardwareConcurrency || 1) : 1;
ort.env.wasm.proxy = true; // keeps the page responsive while the model runs
}
async function download(url, expected, onBytes) {
const parts = [];
let got = 0;
try {
const r = await fetch(url);
if (!r.ok) throw new Error(`HTTP ${r.status}`);
// Not content-length: the CDN compresses, so that header counts different bytes than this stream does.
const reader = r.body.getReader();
for (;;) {
const { done, value } = await reader.read();
if (done) break;
parts.push(value);
got += value.length;
onBytes(Math.min(got / expected, 1));
}
} catch (e) {
throw new Error("Couldn't download the model. Please check your connection and try again.");
}
const out = new Uint8Array(got);
let at = 0;
for (const p of parts) { out.set(p, at); at += p.length; }
return out;
}
// Hugging Face's links can't be cached by the browser, so the model is kept here. Change the name for a new model.
async function modelBytes(say) {
let store = null;
try {
store = await caches.open("indra-3m-v1");
const hit = await store.match(MODEL_URL);
if (hit) return new Uint8Array(await hit.arrayBuffer());
} catch (e) { store = null; }
const bytes = await download(MODEL_URL, MODEL_BYTES, (part) => {
say(`Downloading the model: ${(part * MODEL_BYTES / 1e6).toFixed(0)} of ${(MODEL_BYTES / 1e6).toFixed(0)} MB. This only happens the first time.`, 0.50 + 0.32 * part, 0.82);
});
// Only a complete file is kept, so a cut-off download can't break this device's later visits.
if (bytes.length === MODEL_BYTES) {
try { await store?.put(MODEL_URL, new Response(bytes)); } catch (e) { /* it just downloads again next time */ }
}
return bytes;
}
async function model(say) {
if (session) return session;
say("Getting the forecast engine ready.", 0.29, 0.34);
await loadRuntime();
await download(`${ORT_BASE}ort-wasm-simd-threaded.wasm`, WASM_BYTES, (part) => {
say(`Downloading the forecast engine: ${(part * 100).toFixed(0)}%. This only happens the first time.`, 0.30 + 0.20 * part, 0.50);
});
const bytes = await modelBytes(say);
say("Starting the model on this device.", 0.82, 0.88);
session = await ort.InferenceSession.create(bytes, { executionProviders: ["wasm"] });
return session;
}
async function pass(w, sign) {
const a = assemble(w, sign), sc = scale(a), m = masks(a), R = a.R, S = m.S;
const got = await session.run({
scaled: new ort.Tensor("float32", sc.scaled, [R, a.W]),
observed: new ort.Tensor("bool", a.seen.slice(), [R, a.W]), // a copy: running in a worker takes the original away
time_mask: new ort.Tensor("bool", m.time, [R, 1, S, S]),
variate_mask: new ort.Tensor("bool", m.variate, [S, 1, R, R]),
positions: new ort.Tensor("float32", m.positions, [R, S]),
});
return finish(a, got.quantiles.data, sc);
}
// The averaged forecast of x and -x, the one the model was scored on.
async function quantiles(w, tick) {
const up = await pass(w, 1);
tick();
const down = await pass(w, -1), nT = w.context.length, L = PATCHES_OUT * PS;
tick();
const out = new Float32Array(nT * L * Q);
for (let i = 0; i < nT * L; i++) for (let q = 0; q < Q; q++) out[i * Q + q] = f(0.5 * (up[i * Q + q] + -down[i * Q + Q - 1 - q]));
return out;
}
// ---------------------------------------------------------------- the forecast the page draws
const round2 = (x) => (Number.isFinite(x) ? Math.round(x * 100) / 100 : null);
const iso = (h) => new Date(h * HOUR).toISOString().replace(".000Z", "+00:00");
const windFrom = (u, v) => (Number.isFinite(u) && Number.isFinite(v) && Math.hypot(u, v) >= 0.3 ? Math.round((Math.atan2(-u, -v) * 180 / Math.PI + 360) % 360) : null);
async function fromRaw(station, raw, nowMs, say) {
const obs = buildObs(raw.obs);
if (!obs) throw new Error("This airport has no recent measurements.");
const sources = SOURCES.map((s) => buildSource(s, raw[s.name] || "", obs));
if (sources[0].missing) throw new Error("There's no National Blend forecast for this airport right now.");
let last = -1;
for (let i = 0; i < obs.n; i++) if (Number.isFinite(obs.t[i])) last = i;
if (last < 0) throw new Error("This airport has no recent measurements.");
const now = Math.floor(nowMs / HOUR), origin = Math.min(last + obs.start, now);
if (now - origin > 48) throw new Error("This airport hasn't reported in the last two days.");
await model(say);
let done = 0;
const tick = () => { done++; say("Running the model on this device. Almost there.", 0.88 + 0.03 * done, 0.88 + 0.03 * (done + 1)); };
say("Running the model on this device. Almost there.", 0.88, 0.91);
await new Promise(requestAnimationFrame);
const t0 = performance.now();
const q = await quantiles(window_(obs, sources, origin, station.lon), tick);
const then = await quantiles(window_(obs, sources, origin - 24, station.lon), tick);
const seconds = (performance.now() - t0) / 1000;
const nT = TARGETS.length, L = PATCHES_OUT * PS;
const at = (arr, v, t, l) => arr[(TARGETS.indexOf(v) * L + t) * Q + l];
const forecast = {};
for (const v of ["t", "td", "wspd", "mslp"]) {
forecast[v] = {};
for (const [name, l] of Object.entries(LEVELS)) forecast[v][name] = Array.from({ length: H }, (_, t) => round2(at(q, v, t, l)));
}
forecast.wdir = Array.from({ length: H }, (_, t) => windFrom(at(q, "u", t, MEDIAN), at(q, "v", t, MEDIAN)));
const guidance = {};
for (const s of sources) {
const { out, run } = future(s, origin), g = { run: run == null ? null : iso(run) };
s.vars.forEach((name, c) => { if (name !== "u" && name !== "v") g[name] = Array.from({ length: H }, (_, t) => round2(out[t * s.V + c])); });
guidance[s.name] = g;
}
const i = origin - obs.start, past = [];
for (let k = Math.max(0, i - 47); k <= i; k++) past.push(k);
const observed = { times: past.map((k) => iso(obs.start + k)), wdir: past.map((k) => windFrom(obs.u[k], obs.v[k])) };
for (const v of ["t", "td", "wspd", "mslp", ...PAST]) observed[v] = past.map((k) => round2(obs[v][k]));
return {
station, origin: iso(origin), times: Array.from({ length: H }, (_, t) => iso(origin + 1 + t)), forecast, guidance, observed,
missing_guidance: sources.filter((s) => s.missing).map((s) => s.name),
model: { checkpoint: "Sculptor-AI/Indra-3M", sources: SOURCES.map((s) => s.name), levels: Object.keys(LEVELS).map(Number) },
past_ours: { issued: iso(origin - 24), times: Array.from({ length: H }, (_, t) => iso(origin - 23 + t)),
t: Array.from({ length: H }, (_, t) => round2(at(then, "t", t, MEDIAN))),
td: Array.from({ length: H }, (_, t) => round2(at(then, "td", t, MEDIAN))),
wspd: Array.from({ length: H }, (_, t) => round2(at(then, "wspd", t, MEDIAN))) },
rain: null, past_nws: null, nws: { hourly: [], daily: [] }, generated: new Date(nowMs).toISOString(), device_seconds: Math.round(seconds * 10) / 10,
};
}
// ---------------------------------------------------------------- public data
const sleep = (ms) => new Promise((res) => setTimeout(res, ms));
async function text(url) {
for (let k = 0; k < 3; k++) {
let r = null;
try { r = await fetch(url); } catch (e) { /* retried below */ }
if (r?.ok) {
const t = await r.text();
if (!t.slice(0, 200).includes("Too many requests")) return t;
}
if (k < 2) await sleep(4000 * (k + 1));
}
throw new Error("The weather archive isn't answering right now. Please try again in a minute.");
}
async function fetchRaw(id, say) {
const end = new Date(Date.now() + 864e5), start = new Date(end - (DAYS + 1) * 864e5);
say("Reading this airport's last 90 days of measurements.", 0.02, 0.09);
const q = new URLSearchParams({ station: id, tz: "Etc/UTC", format: "onlycomma", latlon: "no", missing: "empty", trace: "0.0001",
year1: start.getUTCFullYear(), month1: start.getUTCMonth() + 1, day1: start.getUTCDate(),
year2: end.getUTCFullYear(), month2: end.getUTCMonth() + 1, day2: end.getUTCDate() });
for (const v of OBS_VARS) q.append("data", v);
for (const v of ["3", "4"]) q.append("report_type", v);
const raw = { obs: await text(`${IEM}/asos.py?${q}`) };
const z = (d) => d.toISOString().slice(0, 16) + "Z";
let step = 0;
for (const s of SOURCES) {
const from = 0.09 + 0.065 * step++;
say(`Reading ${{ NBS: "National Blend", GFS: "GFS MOS", NAM: "NAM MOS" }[s.name]} forecasts.`, from, from + 0.065);
raw[s.name] = await text(`${IEM}/mos.py?${new URLSearchParams({ station: `K${id}`, model: s.name, format: "csv", sts: z(start), ets: z(end) })}`);
}
return raw;
}
async function nws(lat, lon) {
const out = { hourly: [], daily: [] };
try {
const p = await (await fetch(`https://api.weather.gov/points/${lat.toFixed(4)},${lon.toFixed(4)}`)).json();
const h = await (await fetch(p.properties.forecastHourly)).json();
for (const x of h.properties.periods) {
const spd = (x.windSpeed || "").replace("mph", "").replace("to", " ").trim().split(/\s+/).map(Number).filter(Number.isFinite);
out.hourly.push({ time: x.startTime, t_f: x.temperature, wspd_mph: spd.length ? Math.max(...spd) : null, pop: x.probabilityOfPrecipitation?.value ?? null, short: x.shortForecast });
}
out.updated = h.properties.updateTime;
} catch (e) { out.error = "unavailable"; }
return out;
}
async function forecast(station, say) {
const raw = await fetchRaw(station.id, say);
const [d, n] = await Promise.all([fromRaw(station, raw, Date.now(), say), nws(station.lat, station.lon)]);
d.nws = n;
return d;
}
return { forecast, fromRaw, model };
})();