Spaces:
Running
Running
Deploy the site from pipeline commit 21dc72eed18a
Browse files- assets/duck.js +34 -9
assets/duck.js
CHANGED
|
@@ -22,7 +22,7 @@ export function connect() {
|
|
| 22 |
// would resolve inside the worker, not the page.
|
| 23 |
const base = DATA.startsWith("http") ? DATA : `${location.origin}${DATA}`;
|
| 24 |
const files = Object.keys(m.files).map((f) => [f.replace(/\.parquet$/, ""), `${base}/data/${f}`]);
|
| 25 |
-
return { conn, files, made: new
|
| 26 |
})();
|
| 27 |
return ready;
|
| 28 |
}
|
|
@@ -30,21 +30,46 @@ export function connect() {
|
|
| 30 |
// A view per published file, made the first time a query names it. Making
|
| 31 |
// one reads that file's footer, and every file at once cost megabytes a page
|
| 32 |
// that searches two of them never needed.
|
| 33 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
const db = await connect();
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
}
|
|
|
|
|
|
|
| 40 |
return db.conn;
|
| 41 |
}
|
| 42 |
|
| 43 |
// Rows as plain objects: 64-bit integers as Numbers (every count here is far
|
| 44 |
// below 2^53), dates as ISO strings.
|
| 45 |
export async function query(sql) {
|
| 46 |
-
|
| 47 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
// Arrow hands DATE columns over as epoch milliseconds, not Date objects:
|
| 49 |
// the schema says which columns are dates.
|
| 50 |
const dates = new Set(res.schema.fields.filter((f) => /^Date/.test(String(f.type))).map((f) => f.name));
|
|
|
|
| 22 |
// would resolve inside the worker, not the page.
|
| 23 |
const base = DATA.startsWith("http") ? DATA : `${location.origin}${DATA}`;
|
| 24 |
const files = Object.keys(m.files).map((f) => [f.replace(/\.parquet$/, ""), `${base}/data/${f}`]);
|
| 25 |
+
return { conn, files, made: new Map() };
|
| 26 |
})();
|
| 27 |
return ready;
|
| 28 |
}
|
|
|
|
| 30 |
// A view per published file, made the first time a query names it. Making
|
| 31 |
// one reads that file's footer, and every file at once cost megabytes a page
|
| 32 |
// that searches two of them never needed.
|
| 33 |
+
//
|
| 34 |
+
// ON THE CDN'S OWN ADDRESS. Hugging Face answers every byte-range request for
|
| 35 |
+
// a "resolve" URL with a redirect to its CDN, so a query that reads twenty
|
| 36 |
+
// ranges paid twenty round trips: the explorer took 23-28 s live against 4 s
|
| 37 |
+
// on a local mirror (2026-10-19). The console found this first; like it,
|
| 38 |
+
// each file is resolved once with a HEAD, and again after 30 minutes, since
|
| 39 |
+
// the CDN's signed addresses expire, or when a read fails.
|
| 40 |
+
const RESOLVE_EVERY = 30 * 60 * 1000;
|
| 41 |
+
async function finalUrl(url) {
|
| 42 |
+
if (!/^https?:/.test(url) || url.startsWith(location.origin)) return url;
|
| 43 |
+
try {
|
| 44 |
+
const r = await fetch(url, { method: "HEAD", cache: "no-store" });
|
| 45 |
+
if (r.ok && r.url) return r.url;
|
| 46 |
+
} catch {}
|
| 47 |
+
return url;
|
| 48 |
+
}
|
| 49 |
+
async function views(sql, fresh = false) {
|
| 50 |
const db = await connect();
|
| 51 |
+
const due = db.files.filter(([name]) => new RegExp(`\\b${name}\\b`).test(sql)
|
| 52 |
+
&& (fresh || !db.made.has(name) || Date.now() - db.made.get(name) > RESOLVE_EVERY));
|
| 53 |
+
const urls = await Promise.all(due.map(([, url]) => finalUrl(url)));
|
| 54 |
+
for (const [i, [name]] of due.entries()) {
|
| 55 |
+
await db.conn.query(`CREATE OR REPLACE VIEW ${name} AS SELECT * FROM read_parquet('${urls[i].replace(/'/g, "''")}')`);
|
| 56 |
+
db.made.set(name, Date.now());
|
| 57 |
+
}
|
| 58 |
return db.conn;
|
| 59 |
}
|
| 60 |
|
| 61 |
// Rows as plain objects: 64-bit integers as Numbers (every count here is far
|
| 62 |
// below 2^53), dates as ISO strings.
|
| 63 |
export async function query(sql) {
|
| 64 |
+
let conn = await views(sql), res;
|
| 65 |
+
try {
|
| 66 |
+
res = await conn.query(sql);
|
| 67 |
+
} catch (e) {
|
| 68 |
+
// An expired CDN address: resolve again and retry once.
|
| 69 |
+
if (!/HTTP|40[0-9]|IO Error/i.test(String(e?.message || e))) throw e;
|
| 70 |
+
conn = await views(sql, true);
|
| 71 |
+
res = await conn.query(sql);
|
| 72 |
+
}
|
| 73 |
// Arrow hands DATE columns over as epoch milliseconds, not Date objects:
|
| 74 |
// the schema says which columns are dates.
|
| 75 |
const dates = new Set(res.schema.fields.filter((f) => /^Date/.test(String(f.type))).map((f) => f.name));
|