asingh15's picture
Publish browser-only full-collection task explorer (1-3)
f96162d verified
Raw History Blame Contribute Delete
26.9 kB
(() => {
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __commonJS = (cb, mod) => function __require() {
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
// ../../../../private/tmp/task-viewer-static-runtime/node_modules/sql.js-httpvfs/dist/index.js
var require_dist = __commonJS({
"../../../../private/tmp/task-viewer-static-runtime/node_modules/sql.js-httpvfs/dist/index.js"(exports, module) {
!(function(e, t) {
if ("object" == typeof exports && "object" == typeof module) module.exports = t();
else if ("function" == typeof define && define.amd) define([], t);
else {
var n = t();
for (var r in n) ("object" == typeof exports ? exports : e)[r] = n[r];
}
})(exports, (function() {
return (() => {
"use strict";
var e = { 870: (e2, t2, n2) => {
n2.r(t2), n2.d(t2, { createEndpoint: () => o, expose: () => l, proxy: () => v, proxyMarker: () => r, releaseProxy: () => a, transfer: () => y, transferHandlers: () => c, windowEndpoint: () => g, wrap: () => f });
const r = Symbol("Comlink.proxy"), o = Symbol("Comlink.endpoint"), a = Symbol("Comlink.releaseProxy"), i = Symbol("Comlink.thrown"), s = (e3) => "object" == typeof e3 && null !== e3 || "function" == typeof e3, c = /* @__PURE__ */ new Map([["proxy", { canHandle: (e3) => s(e3) && e3[r], serialize(e3) {
const { port1: t3, port2: n3 } = new MessageChannel();
return l(e3, t3), [n3, [n3]];
}, deserialize: (e3) => (e3.start(), f(e3)) }], ["throw", { canHandle: (e3) => s(e3) && i in e3, serialize({ value: e3 }) {
let t3;
return t3 = e3 instanceof Error ? { isError: true, value: { message: e3.message, name: e3.name, stack: e3.stack } } : { isError: false, value: e3 }, [t3, []];
}, deserialize(e3) {
if (e3.isError) throw Object.assign(new Error(e3.value.message), e3.value);
throw e3.value;
} }]]);
function l(e3, t3 = self) {
t3.addEventListener("message", (function n3(r2) {
if (!r2 || !r2.data) return;
const { id: o2, type: a2, path: s2 } = Object.assign({ path: [] }, r2.data), c2 = (r2.data.argumentList || []).map(w);
let f2;
try {
const t4 = s2.slice(0, -1).reduce(((e4, t5) => e4[t5]), e3), n4 = s2.reduce(((e4, t5) => e4[t5]), e3);
switch (a2) {
case 0:
f2 = n4;
break;
case 1:
t4[s2.slice(-1)[0]] = w(r2.data.value), f2 = true;
break;
case 2:
f2 = n4.apply(t4, c2);
break;
case 3:
f2 = v(new n4(...c2));
break;
case 4:
{
const { port1: t5, port2: n5 } = new MessageChannel();
l(e3, n5), f2 = y(t5, [t5]);
}
break;
case 5:
f2 = void 0;
}
} catch (e4) {
f2 = { value: e4, [i]: 0 };
}
Promise.resolve(f2).catch(((e4) => ({ value: e4, [i]: 0 }))).then(((e4) => {
const [r3, i2] = b(e4);
t3.postMessage(Object.assign(Object.assign({}, r3), { id: o2 }), i2), 5 === a2 && (t3.removeEventListener("message", n3), u(t3));
}));
})), t3.start && t3.start();
}
function u(e3) {
(function(e4) {
return "MessagePort" === e4.constructor.name;
})(e3) && e3.close();
}
function f(e3, t3) {
return d(e3, [], t3);
}
function p(e3) {
if (e3) throw new Error("Proxy has been released and is not useable");
}
function d(e3, t3 = [], n3 = function() {
}) {
let r2 = false;
const i2 = new Proxy(n3, { get(n4, o2) {
if (p(r2), o2 === a) return () => E(e3, { type: 5, path: t3.map(((e4) => e4.toString())) }).then((() => {
u(e3), r2 = true;
}));
if ("then" === o2) {
if (0 === t3.length) return { then: () => i2 };
const n5 = E(e3, { type: 0, path: t3.map(((e4) => e4.toString())) }).then(w);
return n5.then.bind(n5);
}
return d(e3, [...t3, o2]);
}, set(n4, o2, a2) {
p(r2);
const [i3, s2] = b(a2);
return E(e3, { type: 1, path: [...t3, o2].map(((e4) => e4.toString())), value: i3 }, s2).then(w);
}, apply(n4, a2, i3) {
p(r2);
const s2 = t3[t3.length - 1];
if (s2 === o) return E(e3, { type: 4 }).then(w);
if ("bind" === s2) return d(e3, t3.slice(0, -1));
const [c2, l2] = m(i3);
return E(e3, { type: 2, path: t3.map(((e4) => e4.toString())), argumentList: c2 }, l2).then(w);
}, construct(n4, o2) {
p(r2);
const [a2, i3] = m(o2);
return E(e3, { type: 3, path: t3.map(((e4) => e4.toString())), argumentList: a2 }, i3).then(w);
} });
return i2;
}
function m(e3) {
const t3 = e3.map(b);
return [t3.map(((e4) => e4[0])), (n3 = t3.map(((e4) => e4[1])), Array.prototype.concat.apply([], n3))];
var n3;
}
const h = /* @__PURE__ */ new WeakMap();
function y(e3, t3) {
return h.set(e3, t3), e3;
}
function v(e3) {
return Object.assign(e3, { [r]: true });
}
function g(e3, t3 = self, n3 = "*") {
return { postMessage: (t4, r2) => e3.postMessage(t4, n3, r2), addEventListener: t3.addEventListener.bind(t3), removeEventListener: t3.removeEventListener.bind(t3) };
}
function b(e3) {
for (const [t3, n3] of c) if (n3.canHandle(e3)) {
const [r2, o2] = n3.serialize(e3);
return [{ type: 3, name: t3, value: r2 }, o2];
}
return [{ type: 0, value: e3 }, h.get(e3) || []];
}
function w(e3) {
switch (e3.type) {
case 3:
return c.get(e3.name).deserialize(e3.value);
case 0:
return e3.value;
}
}
function E(e3, t3, n3) {
return new Promise(((r2) => {
const o2 = new Array(4).fill(0).map((() => Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(16))).join("-");
e3.addEventListener("message", (function t4(n4) {
n4.data && n4.data.id && n4.data.id === o2 && (e3.removeEventListener("message", t4), r2(n4.data));
})), e3.start && e3.start(), e3.postMessage(Object.assign({ id: o2 }, t3), n3);
}));
}
}, 162: function(e2, t2, n2) {
var r = this && this.__createBinding || (Object.create ? function(e3, t3, n3, r2) {
void 0 === r2 && (r2 = n3), Object.defineProperty(e3, r2, { enumerable: true, get: function() {
return t3[n3];
} });
} : function(e3, t3, n3, r2) {
void 0 === r2 && (r2 = n3), e3[r2] = t3[n3];
}), o = this && this.__setModuleDefault || (Object.create ? function(e3, t3) {
Object.defineProperty(e3, "default", { enumerable: true, value: t3 });
} : function(e3, t3) {
e3.default = t3;
}), a = this && this.__importStar || function(e3) {
if (e3 && e3.__esModule) return e3;
var t3 = {};
if (null != e3) for (var n3 in e3) "default" !== n3 && Object.prototype.hasOwnProperty.call(e3, n3) && r(t3, e3, n3);
return o(t3, e3), t3;
};
Object.defineProperty(t2, "__esModule", { value: true }), t2.createDbWorker = void 0;
const i = a(n2(870));
async function s(e3) {
if (e3.data && "eval" === e3.data.action) {
const t3 = new Int32Array(e3.data.notify, 0, 2), n3 = new Uint8Array(e3.data.notify, 8);
let r2;
try {
r2 = { ok: await u(e3.data.request) };
} catch (t4) {
console.error("worker request error", e3.data.request, t4), r2 = { err: String(t4) };
}
const o2 = new TextEncoder().encode(JSON.stringify(r2));
n3.set(o2, 0), t3[1] = o2.length, Atomics.notify(t3, 0);
}
}
function c(e3) {
if ("BODY" === e3.tagName) return "body";
const t3 = [];
for (; e3.parentElement && "BODY" !== e3.tagName; ) {
if (e3.id) {
t3.unshift("#" + e3.id);
break;
}
{
let n3 = 1, r2 = e3;
for (; r2.previousElementSibling; ) r2 = r2.previousElementSibling, n3++;
t3.unshift(e3.tagName.toLowerCase() + ":nth-child(" + n3 + ")");
}
e3 = e3.parentElement;
}
return t3.join(" > ");
}
function l(e3) {
return Object.keys(e3);
}
async function u(e3) {
if (console.log("dom vtable request", e3), "select" === e3.type) return [...document.querySelectorAll(e3.selector)].map(((t3) => {
const n3 = {};
for (const r2 of e3.columns) "selector" === r2 ? n3.selector = c(t3) : "parent" === r2 ? t3.parentElement && (n3.parent = t3.parentElement ? c(t3.parentElement) : null) : "idx" === r2 || (n3[r2] = t3[r2]);
return n3;
}));
if ("insert" === e3.type) {
if (!e3.value.parent) throw Error('"parent" column must be set when inserting');
const t3 = document.querySelectorAll(e3.value.parent);
if (0 === t3.length) throw Error(`Parent element ${e3.value.parent} could not be found`);
if (t3.length > 1) throw Error(`Parent element ${e3.value.parent} ambiguous (${t3.length} results)`);
const n3 = t3[0];
if (!e3.value.tagName) throw Error("tagName must be set for inserting");
const r2 = document.createElement(e3.value.tagName);
for (const t4 of l(e3.value)) if (null !== e3.value[t4]) {
if ("tagName" === t4 || "parent" === t4) continue;
if ("idx" === t4 || "selector" === t4) throw Error(`${t4} can't be set`);
r2[t4] = e3.value[t4];
}
return n3.appendChild(r2), null;
}
if ("update" === e3.type) {
const t3 = document.querySelector(e3.value.selector);
if (!t3) throw Error(`Element ${e3.value.selector} not found!`);
const n3 = [];
for (const r2 of l(e3.value)) {
const o2 = e3.value[r2];
if ("parent" !== r2) {
if ("idx" !== r2 && "selector" !== r2 && o2 !== t3[r2]) {
if (console.log("SETTING ", r2, t3[r2], "->", o2), "tagName" === r2) throw Error("can't change tagName");
n3.push(r2);
}
} else if (o2 !== c(t3.parentElement)) {
const e4 = document.querySelectorAll(o2);
if (1 !== e4.length) throw Error(`Invalid target parent: found ${e4.length} matches`);
e4[0].appendChild(t3);
}
}
for (const r2 of n3) t3[r2] = e3.value[r2];
return null;
}
throw Error(`unknown request ${e3.type}`);
}
i.transferHandlers.set("WORKERSQLPROXIES", { canHandle: (e3) => false, serialize(e3) {
throw Error("no");
}, deserialize: (e3) => (e3.start(), i.wrap(e3)) }), t2.createDbWorker = async function(e3, t3, n3, r2 = 1 / 0) {
const o2 = new Worker(t3), a2 = i.wrap(o2), c2 = await a2.SplitFileHttpDatabase(n3, e3, void 0, r2);
return o2.addEventListener("message", s), { db: c2, worker: a2, configs: e3 };
};
}, 432: function(e2, t2, n2) {
var r = this && this.__createBinding || (Object.create ? function(e3, t3, n3, r2) {
void 0 === r2 && (r2 = n3), Object.defineProperty(e3, r2, { enumerable: true, get: function() {
return t3[n3];
} });
} : function(e3, t3, n3, r2) {
void 0 === r2 && (r2 = n3), e3[r2] = t3[n3];
}), o = this && this.__exportStar || function(e3, t3) {
for (var n3 in e3) "default" === n3 || Object.prototype.hasOwnProperty.call(t3, n3) || r(t3, e3, n3);
};
Object.defineProperty(t2, "__esModule", { value: true }), o(n2(162), t2);
} }, t = {};
function n(r) {
var o = t[r];
if (void 0 !== o) return o.exports;
var a = t[r] = { exports: {} };
return e[r].call(a.exports, a, a.exports, n), a.exports;
}
return n.d = (e2, t2) => {
for (var r in t2) n.o(t2, r) && !n.o(e2, r) && Object.defineProperty(e2, r, { enumerable: true, get: t2[r] });
}, n.o = (e2, t2) => Object.prototype.hasOwnProperty.call(e2, t2), n.r = (e2) => {
"undefined" != typeof Symbol && Symbol.toStringTag && Object.defineProperty(e2, Symbol.toStringTag, { value: "Module" }), Object.defineProperty(e2, "__esModule", { value: true });
}, n(432);
})();
}));
}
});
// scripts/task_viewer_data_client.js
var import_sql = __toESM(require_dist());
(() => {
"use strict";
const base = new URL(".", document.currentScript?.src || location.href);
const dataBase = new URL("data/", base);
const fields = "id,title,preview,source,group_sources,family,difficulty_tier,source_split_status,has_native_rubric,has_reference_solution,has_verifier,curation_status";
const booleanFields = ["has_native_rubric", "has_reference_solution", "has_verifier"];
const collections = /* @__PURE__ */ new Set(["qwen_core", "tasks", "foundational", "quarantine"]);
const evidenceBits = { rubric: 1, reference: 2, verifier: 4 };
let readyPromise, metadata, matrix, view, database, fetchBytes = 0;
const textCache = /* @__PURE__ */ new Map();
const summaryCache = /* @__PURE__ */ new Map();
const abort = (signal) => {
if (signal?.aborted) throw new DOMException("The request was cancelled.", "AbortError");
};
function objects(result) {
if (!result?.length) return [];
const { columns, values } = result[0];
return values.map((row) => Object.fromEntries(columns.map((column, index) => [column, row[index]])));
}
function databaseError(error) {
if (/disk I\/O|IOERR|memory/i.test(error.message)) return new Error("This browser session reached its 128 MB search download budget. Reload the page, then use more specific search words.");
return error;
}
async function query(sql, parameters2 = []) {
try {
return objects(await database.db.exec(sql, parameters2));
} catch (error) {
throw databaseError(error);
}
}
async function initialize() {
if (!readyPromise) readyPromise = (async () => {
const response = await fetch(new URL("static_index.json", dataBase));
if (!response.ok) throw new Error("The static search index is unavailable. Please retry.");
metadata = await response.json();
const filterResponse = await fetch(new URL(metadata.filter_matrix.file, dataBase));
if (!filterResponse.ok) throw new Error("The task filter index could not be loaded.");
if (typeof DecompressionStream === "undefined") throw new Error("This viewer needs a current browser with gzip decompression support.");
const packed = await filterResponse.arrayBuffer();
fetchBytes += packed.byteLength;
const buffer = await new Response(new Blob([packed]).stream().pipeThrough(new DecompressionStream("gzip"))).arrayBuffer();
if (buffer.byteLength !== metadata.filter_matrix.bytes) throw new Error("The task filter index is incomplete. Reload the page.");
const digest = Array.from(new Uint8Array(await crypto.subtle.digest("SHA-256", buffer))).map((x) => x.toString(16).padStart(2, "0")).join("");
if (digest !== metadata.filter_matrix.sha256) throw new Error("The task filter index failed its integrity check. Reload the page.");
matrix = new Uint8Array(buffer);
view = new DataView(buffer);
const config = { ...metadata.sqlite };
if (config.serverMode === "chunked") config.urlPrefix = new URL(config.urlPrefix, dataBase).href;
else config.url = new URL(config.url, dataBase).href;
database = await (0, import_sql.createDbWorker)(
[{ from: "inline", config }],
new URL("sql-worker.js", base).href,
new URL("vendor/sqlite/sql-wasm.wasm", base).href,
128 * 1024 * 1024
);
const feature = await query("SELECT sqlite_compileoption_used('ENABLE_FTS5') AS enabled");
if (!feature[0]?.enabled) throw new Error("The browser search engine does not support this full-text index.");
return metadata;
})().catch((error) => {
readyPromise = null;
throw error;
});
return readyPromise;
}
function parameters(input = {}) {
const p = input instanceof URLSearchParams ? Object.fromEntries(input) : input;
const result = { collection: p.collection || "qwen_core", q: (p.q || "").trim(), source: p.source || "", family: p.family || "", difficulty: p.difficulty || "", split: p.split || "", evidence: p.evidence || "", offset: Number(p.offset || 0), limit: Number(p.limit || 20) };
if (!collections.has(result.collection)) throw new Error("Choose a valid task collection.");
if (!Number.isSafeInteger(result.offset) || result.offset < 0 || !Number.isSafeInteger(result.limit) || result.limit < 1 || result.limit > 100) throw new Error("The requested task page is invalid.");
if (result.q.length > 500) throw new Error("Use a search query of at most 500 characters.");
if (result.evidence && !evidenceBits[result.evidence]) throw new Error("Choose rubric, reference, or verifier evidence.");
return result;
}
function range(collection) {
const curated = metadata.counts.tasks, foundations = metadata.counts.foundational;
if (collection === "foundational") return [1, foundations];
if (collection === "qwen_core") return [foundations + 1, curated];
if (collection === "quarantine") return [curated + 1, metadata.records];
return [1, curated];
}
function selection(p) {
const codes = { family: "family", difficulty: "difficulty_tier", split: "source_split_status" };
const wanted = {};
for (const [key, label] of Object.entries(codes)) {
wanted[key] = p[key] ? metadata.labels[label].indexOf(p[key]) : -1;
if (p[key] && wanted[key] < 0) return { none: true };
}
let sourceMask = 0;
if (p.source) {
const code = metadata.labels.source.indexOf(p.source);
if (code < 0) return { none: true };
sourceMask = 1 << code >>> 0;
}
return { ...wanted, sourceMask, evidence: evidenceBits[p.evidence] || 0 };
}
function matches(id, filter) {
const at = (id - 1) * 8;
return (!filter.sourceMask || (view.getUint32(at, true) & filter.sourceMask) !== 0) && (filter.family < 0 || matrix[at + 4] === filter.family) && (filter.difficulty < 0 || matrix[at + 5] === filter.difficulty) && (filter.split < 0 || matrix[at + 6] === filter.split) && (!filter.evidence || (matrix[at + 7] & filter.evidence) !== 0);
}
function expression(q) {
const words = q.match(/[\p{L}\p{N}_]+/gu) || [];
if (!words.length) throw new Error("Enter at least one word or number to search.");
if (words.length > 30) throw new Error("Use at most 30 search words.");
return words.map((word) => '"' + word.replaceAll('"', '""') + '"').join(" AND ");
}
async function fulltext(q, lower, upper) {
const match = expression(q), key = JSON.stringify([match, lower, upper]);
if (textCache.has(key)) return textCache.get(key);
const code = `const statement=db.prepare('SELECT rowid FROM search WHERE search MATCH ? AND rowid>=? AND rowid<=? ORDER BY rowid');
statement.bind(${JSON.stringify([match, lower, upper])});
const ids=new Uint32Array(${upper - lower + 1});let length=0;
try{while(statement.step())ids[length++]=statement.get()[0];}finally{statement.free();}
return ids.slice(0,length);`;
let result;
try {
result = await database.worker.evalCode(code);
} catch (error) {
throw databaseError(error);
}
if (textCache.size >= 3) textCache.delete(textCache.keys().next().value);
textCache.set(key, result);
return result;
}
async function matchingIds(p) {
await initialize();
const filter = selection(p), [lower, upper] = range(p.collection);
if (filter.none) return new Uint32Array();
const ids = new Uint32Array(upper - lower + 1);
let length = 0;
for (let id = lower; id <= upper; id++) if (matches(id, filter)) ids[length++] = id;
if (!length) return new Uint32Array();
if (p.q) {
const found = await fulltext(p.q, ids[0], ids[length - 1]);
return found.filter((id) => matches(id, filter));
}
return ids.subarray(0, length);
}
async function summaries(ids) {
if (!ids.length) return [];
const missing = Array.from(ids).filter((id) => !summaryCache.has(id));
if (missing.length) {
const rows = await query(`SELECT ${fields} FROM tasks WHERE id IN (${missing.map(() => "?").join(",")}) ORDER BY id`, missing);
for (const row of rows) {
row.group_sources = JSON.parse(row.group_sources);
for (const field of booleanFields) row[field] = Boolean(row[field]);
summaryCache.set(row.id, row);
}
if (summaryCache.size > 1e3) {
const keep = new Set(ids);
for (const key of summaryCache.keys()) if (!keep.has(key)) summaryCache.delete(key);
}
}
return Array.from(ids, (id) => summaryCache.get(id));
}
async function search(input = {}) {
const p = parameters(input), ids = await matchingIds(p);
return { rows: await summaries(ids.subarray(p.offset, p.offset + p.limit)), total: ids.length, offset: p.offset, limit: p.limit };
}
async function random(input = {}) {
const ids = await matchingIds(parameters(input));
if (!ids.length) throw new Error("No tasks match these filters.");
return { id: ids[Math.floor(Math.random() * ids.length)], total: ids.length };
}
async function detailLocator(id) {
await initialize();
id = Number(id);
if (!Number.isSafeInteger(id) || id < 1 || id > metadata.records) throw new Error("This task does not exist in this dataset revision.");
const rows = await query("SELECT t.id,t.row_in_shard,s.path AS parquet_file,s.sha256,s.bytes FROM tasks t JOIN shards s ON s.id=t.shard_id WHERE t.id=?", [id]);
if (!rows.length) throw new Error("This task does not exist in this dataset revision.");
return { ...rows[0], dataset: metadata.dataset, revision: metadata.revision };
}
async function request(path, { signal } = {}) {
abort(signal);
const url = new URL(path, location.href);
let response;
if (url.pathname === "/api/tasks") response = await search(url.searchParams);
else if (url.pathname === "/api/random") response = await random(url.searchParams);
else if (url.pathname === "/api/health") {
await initialize();
response = { status: "ready", dataset: metadata.dataset, revision: metadata.revision, index: metadata, mode: "static" };
} else if (/^\/api\/tasks\/\d+$/.test(url.pathname)) {
const locator = await detailLocator(url.pathname.split("/").pop());
abort(signal);
const record = await window.TaskParquet.load(locator, { signal });
response = { ...record, id: locator.id, _viewer: { ...record._viewer || {}, dataset: locator.dataset, revision: locator.revision, parquet_file: locator.parquet_file, row_in_shard: locator.row_in_shard } };
} else throw new Error("Unknown viewer request.");
abort(signal);
return response;
}
async function stats() {
await initialize();
return { ...await database.worker.getStats(), filterFetchedBytes: fetchBytes, filterMemoryBytes: matrix.byteLength, mode: "static" };
}
window.TaskDataClient = { request, search, random, detailLocator, stats, initialize };
window.TaskData = window.TaskDataClient;
})();
})();