clyxconsultingdev commited on
Commit
ef8fbc5
·
verified ·
1 Parent(s): 4c87e37

Deploy the site from pipeline commit 21dc72eed18a

Browse files
Files changed (1) hide show
  1. assets/duck.js +34 -9
assets/duck.js CHANGED
@@ -22,7 +22,7 @@ export function connect() {
22
  // would resolve inside the worker, not the page.
23
  const base = DATA.startsWith("http") ? DATA : `${location.origin}${DATA}`;
24
  const files = Object.keys(m.files).map((f) => [f.replace(/\.parquet$/, ""), `${base}/data/${f}`]);
25
- return { conn, files, made: new Set() };
26
  })();
27
  return ready;
28
  }
@@ -30,21 +30,46 @@ export function connect() {
30
  // A view per published file, made the first time a query names it. Making
31
  // one reads that file's footer, and every file at once cost megabytes a page
32
  // that searches two of them never needed.
33
- async function views(sql) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
34
  const db = await connect();
35
- for (const [name, url] of db.files)
36
- if (!db.made.has(name) && new RegExp(`\\b${name}\\b`).test(sql)) {
37
- await db.conn.query(`CREATE OR REPLACE VIEW ${name} AS SELECT * FROM read_parquet('${url}')`);
38
- db.made.add(name);
39
- }
 
 
40
  return db.conn;
41
  }
42
 
43
  // Rows as plain objects: 64-bit integers as Numbers (every count here is far
44
  // below 2^53), dates as ISO strings.
45
  export async function query(sql) {
46
- const conn = await views(sql);
47
- const res = await conn.query(sql);
 
 
 
 
 
 
 
48
  // Arrow hands DATE columns over as epoch milliseconds, not Date objects:
49
  // the schema says which columns are dates.
50
  const dates = new Set(res.schema.fields.filter((f) => /^Date/.test(String(f.type))).map((f) => f.name));
 
22
  // would resolve inside the worker, not the page.
23
  const base = DATA.startsWith("http") ? DATA : `${location.origin}${DATA}`;
24
  const files = Object.keys(m.files).map((f) => [f.replace(/\.parquet$/, ""), `${base}/data/${f}`]);
25
+ return { conn, files, made: new Map() };
26
  })();
27
  return ready;
28
  }
 
30
  // A view per published file, made the first time a query names it. Making
31
  // one reads that file's footer, and every file at once cost megabytes a page
32
  // that searches two of them never needed.
33
+ //
34
+ // ON THE CDN'S OWN ADDRESS. Hugging Face answers every byte-range request for
35
+ // a "resolve" URL with a redirect to its CDN, so a query that reads twenty
36
+ // ranges paid twenty round trips: the explorer took 23-28 s live against 4 s
37
+ // on a local mirror (2026-10-19). The console found this first; like it,
38
+ // each file is resolved once with a HEAD, and again after 30 minutes, since
39
+ // the CDN's signed addresses expire, or when a read fails.
40
+ const RESOLVE_EVERY = 30 * 60 * 1000;
41
+ async function finalUrl(url) {
42
+ if (!/^https?:/.test(url) || url.startsWith(location.origin)) return url;
43
+ try {
44
+ const r = await fetch(url, { method: "HEAD", cache: "no-store" });
45
+ if (r.ok && r.url) return r.url;
46
+ } catch {}
47
+ return url;
48
+ }
49
+ async function views(sql, fresh = false) {
50
  const db = await connect();
51
+ const due = db.files.filter(([name]) => new RegExp(`\\b${name}\\b`).test(sql)
52
+ && (fresh || !db.made.has(name) || Date.now() - db.made.get(name) > RESOLVE_EVERY));
53
+ const urls = await Promise.all(due.map(([, url]) => finalUrl(url)));
54
+ for (const [i, [name]] of due.entries()) {
55
+ await db.conn.query(`CREATE OR REPLACE VIEW ${name} AS SELECT * FROM read_parquet('${urls[i].replace(/'/g, "''")}')`);
56
+ db.made.set(name, Date.now());
57
+ }
58
  return db.conn;
59
  }
60
 
61
  // Rows as plain objects: 64-bit integers as Numbers (every count here is far
62
  // below 2^53), dates as ISO strings.
63
  export async function query(sql) {
64
+ let conn = await views(sql), res;
65
+ try {
66
+ res = await conn.query(sql);
67
+ } catch (e) {
68
+ // An expired CDN address: resolve again and retry once.
69
+ if (!/HTTP|40[0-9]|IO Error/i.test(String(e?.message || e))) throw e;
70
+ conn = await views(sql, true);
71
+ res = await conn.query(sql);
72
+ }
73
  // Arrow hands DATE columns over as epoch milliseconds, not Date objects:
74
  // the schema says which columns are dates.
75
  const dates = new Set(res.schema.fields.filter((f) => /^Date/.test(String(f.type))).map((f) => f.name));