Spaces:
Running
Running
Download core.js from thelamapi/Next-Trace: direct link, hf CLI and curl.
- Browser
- Download file 8.21 kB
-
https://huggingface.co/spaces/thelamapi/Next-Trace/resolve/main/core.js
- Command line
-
hf download hf://spaces/thelamapi/Next-Trace/core.js
-
curl -L -o core.js https://huggingface.co/spaces/thelamapi/Next-Trace/resolve/main/core.js
8.21 kB
| /* Next Trace compute core: line-by-line port of trace_core.py. */ | |
| (function (root, factory) { | |
| if (typeof module === "object" && module.exports) module.exports = factory(); | |
| else root.Core = factory(); | |
| })(typeof self !== "undefined" ? self : this, function () { | |
| "use strict"; | |
| const ASSETS = ["model.json"]; | |
| const STOP = new Set(`a an the of in on at to for from by with and or but is are was were be been being it its this that these those as | |
| which who whom whose what when where why how do does did has have had not no than then there their they he she his her | |
| we you your i me my our us can could would should will may might also into over under about after before between during | |
| bir ve ile de da mi mı mu mü ne bu şu o için gibi daha en çok olarak olan olduğu ise ki ama veya hangi kim kimin neden | |
| nasıl nerede ne zaman kaç her bazı tüm diğer sonra önce kadar göre üzerinde arasında tarafından`.split(/\s+/)); | |
| const WH_TIME = new Set(["when", "year", "zaman", "yıl", "yılında", "tarih", "century", "yüzyıl"]); | |
| const WH_NUM = new Set(["many", "much", "kaç", "number", "percent", "yüzde", "sayı"]); | |
| const WH_WHO = new Set(["who", "whom", "whose", "kim", "kimin", "kime", "kimdir"]); | |
| const SENT_RE = /(?<=[.!?])\s+(?=[A-ZÇĞİÖŞÜ0-9"“(])/g; | |
| const ALNUM = /[\p{L}\p{N}]/u; | |
| const lowerTr = s => s.replace(/I/g, "ı").replace(/İ/g, "i").toLowerCase(); | |
| function splitSentences(text) { | |
| text = text.replace(//g, " "); | |
| const spans = []; let start = 0, m; | |
| SENT_RE.lastIndex = 0; | |
| while ((m = SENT_RE.exec(text)) !== null) { | |
| if (text.slice(start, m.index).trim()) spans.push([start, m.index]); | |
| start = m.index + m[0].length; | |
| } | |
| if (text.slice(start).trim()) spans.push([start, text.length]); | |
| return spans; | |
| } | |
| function words(text) { | |
| const out = []; let cur = ""; | |
| for (const ch of lowerTr(text)) { if (ALNUM.test(ch)) cur += ch; else if (cur) { out.push(cur); cur = ""; } } | |
| if (cur) out.push(cur); | |
| return out; | |
| } | |
| const stems = text => words(text).filter(w => !STOP.has(w)).map(w => Array.from(w).slice(0, 5).join("")); | |
| function bm25(q, sents, k1 = 1.2, b = 0.75) { | |
| const n = sents.length; let tot = 0; for (const s of sents) tot += s.length; | |
| const avg = tot / Math.max(n, 1), df = new Map(); | |
| for (const s of sents) for (const t of new Set(s)) df.set(t, (df.get(t) || 0) + 1); | |
| const idf = new Map(); for (const [t, d] of df) idf.set(t, Math.log(1.0 + (n - d + 0.5) / (d + 0.5))); | |
| const scores = sents.map(s => { | |
| const tf = new Map(); for (const t of s) tf.set(t, (tf.get(t) || 0) + 1); | |
| let sc = 0.0; | |
| for (const t of q) if (tf.has(t)) { const f = tf.get(t); sc += idf.get(t) * f * (k1 + 1) / (f + k1 * (1 - b + b * s.length / Math.max(avg, 1e-9))); } | |
| return sc; | |
| }); | |
| return [scores, idf]; | |
| } | |
| function charNgrams(text, n = 4) { | |
| const t = words(text).join(" "), out = new Set(); | |
| for (let i = 0; i < Math.max(0, t.length - n + 1); i++) out.add(t.slice(i, i + n)); | |
| return out; | |
| } | |
| const isUpper = c => c !== c.toLowerCase() && c === c.toUpperCase(); | |
| function sentenceFeatures(question, sents) { | |
| const qSt = Array.from(new Set(stems(question))), qWords = new Set(words(question)); | |
| const sSt = sents.map(stems); | |
| const [scores, idf] = bm25(qSt, sSt); | |
| const mx = scores.length ? Math.max(...scores) : 0.0; | |
| const order = scores.map((_, i) => i).sort((a, b) => (scores[b] - scores[a]) || (a - b)); | |
| const rank = new Map(order.map((i, r) => [i, r])); | |
| const qIdf = qSt.map(t => (idf.has(t) ? idf.get(t) : Math.log(1.0 + (sents.length + 0.5) / 0.5))); | |
| const qIdfSum = qIdf.reduce((a, b) => a + b, 0) || 1.0, qMax = qIdf.length ? Math.max(...qIdf) : 0; | |
| const qNg = charNgrams(question); | |
| const has = set => [...set].some(w => qWords.has(w)); | |
| const wantTime = has(WH_TIME), wantNum = has(WH_NUM), wantWho = has(WH_WHO); | |
| const docSt = new Set(); for (const st of sSt) for (const t of st) docSt.add(t); | |
| const docMissing = qSt.filter(t => !docSt.has(t)).length / Math.max(qSt.length, 1); | |
| const qNums = words(question).filter(w => w && /^[0-9]+$/.test(w)); | |
| const bigQ = new Set(); for (let j = 0; j + 1 < qSt.length; j++) bigQ.add(qSt[j] + "\u0000" + qSt[j + 1]); | |
| const feats = sents.map((s, i) => { | |
| const ss = new Set(sSt[i]); | |
| const matched = [], missing = []; | |
| qSt.forEach((t, j) => (ss.has(t) ? matched : missing).push(qIdf[j])); | |
| const ng = charNgrams(s); let inter = 0; for (const g of qNg) if (ng.has(g)) inter++; | |
| const uni = qNg.size + ng.size - inter, jac = inter / Math.max(uni, 1); | |
| const bigS = new Set(); for (let j = 0; j + 1 < sSt[i].length; j++) bigS.add(sSt[i][j] + "\u0000" + sSt[i][j + 1]); | |
| let bi = 0; for (const g of bigQ) if (bigS.has(g)) bi++; | |
| const hasNum = /[0-9]/.test(s); | |
| const hasCap = s.split(/\s+/).filter(Boolean).slice(1).some(w => isUpper(w[0])); | |
| const nb = [i - 1, i + 1].filter(j => j >= 0 && j < sents.length).map(j => scores[j]); | |
| return [mx > 0 ? scores[i] / mx : 0.0, rank.get(i) / Math.max(sents.length - 1, 1), matched.length / Math.max(qSt.length, 1), | |
| matched.reduce((a, b) => a + b, 0) / qIdfSum, matched.length && qIdf.length ? Math.max(...matched) / qMax : 0.0, | |
| missing.length && qIdf.length ? Math.max(...missing) / qMax : 0.0, jac, Math.min(bi, 3) / 3.0, | |
| (wantTime || wantNum) && hasNum ? 1.0 : 0.0, wantWho && hasCap ? 1.0 : 0.0, (wantTime || wantNum) && !hasNum ? 1.0 : 0.0, | |
| Math.log(1 + sSt[i].length) / 4.0, i / Math.max(sents.length - 1, 1), nb.length && mx > 0 ? Math.max(...nb) / mx : 0.0, | |
| Math.log(1 + mx) / 3.0, Math.log(1 + sents.length) / 4.0, docMissing, | |
| qNums.length && qNums.some(n => !new Set(words(s)).has(n)) ? 1.0 : 0.0]; | |
| }); | |
| return [feats, scores]; | |
| } | |
| const sigmoid = z => 1.0 / (1.0 + Math.exp(-Math.max(-50.0, Math.min(50.0, z)))); | |
| function scoreDocument(ctx, question, document) { | |
| const spans = splitSentences(document), sents = spans.map(([a, b]) => document.slice(a, b)); | |
| if (!sents.length) return { spans: [], probs: [], bm25: [], answerable: 0.0, abstain: true, evidence: [] }; | |
| const [feats, scores] = sentenceFeatures(question, sents); | |
| const probs = feats.map(f => sigmoid(NLRT.forward(ctx.m.nets.sent, NLRT.T([f.length], Float64Array.from(f))).data[0])); | |
| const order = probs.map((_, i) => i).sort((a, b) => (probs[b] - probs[a]) || (a - b)); | |
| const top = order[0], p2 = order.length > 1 ? probs[order[1]] : 0.0; | |
| const df = [probs[top], probs[top] - p2, feats[top][2], feats[top][3], feats[top][5], feats[top][10], feats[top][14], feats[top][16], feats[top][17]]; | |
| const w = ctx.m.tensors.ans_w.data; let z = 0; for (let i = 0; i < df.length; i++) z += w[i] * df[i]; | |
| const ans = sigmoid(z + ctx.m.tensors.ans_b.data[0]), th = ctx.m.meta.thresholds; | |
| const abstain = ans < th.answerable; | |
| const evidence = abstain ? [] : order.slice(0, 2).filter(i => i === top || probs[i] >= th.second_sentence); | |
| return { spans, probs, bm25: scores, answerable: ans, abstain, evidence }; | |
| } | |
| function bm25Baseline(ctx, question, document) { | |
| const spans = splitSentences(document), sents = spans.map(([a, b]) => document.slice(a, b)); | |
| if (!sents.length) return { spans: [], abstain: true, evidence: [], bm25: [] }; | |
| const [scores] = bm25(Array.from(new Set(stems(question))), sents.map(stems)); | |
| let best = 0; for (let i = 1; i < scores.length; i++) if (scores[i] > scores[best]) best = i; | |
| const abstain = scores[best] < ctx.m.meta.thresholds.bm25; | |
| return { spans, abstain, evidence: abstain ? [] : [best], bm25: scores }; | |
| } | |
| function init(assets) { return { m: NLRT.decode(assets["model.json"]) }; } | |
| function runCase(ctx, input) { | |
| const r = scoreDocument(ctx, input.question, input.document), b = bm25Baseline(ctx, input.question, input.document); | |
| return { spans: r.spans, probs: r.probs, answerable: r.answerable, abstain: r.abstain, evidence: r.evidence, bm25_abstain: b.abstain, bm25_evidence: b.evidence }; | |
| } | |
| return { ASSETS, init, splitSentences, scoreDocument, bm25Baseline, runCase, words, stems }; | |
| }); | |