Spaces:
Running
Running
File size: 8,205 Bytes
c07bca1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 | /* Next Trace compute core: line-by-line port of trace_core.py. */
(function (root, factory) {
if (typeof module === "object" && module.exports) module.exports = factory();
else root.Core = factory();
})(typeof self !== "undefined" ? self : this, function () {
"use strict";
const ASSETS = ["model.json"];
const STOP = new Set(`a an the of in on at to for from by with and or but is are was were be been being it its this that these those as
which who whom whose what when where why how do does did has have had not no than then there their they he she his her
we you your i me my our us can could would should will may might also into over under about after before between during
bir ve ile de da mi mı mu mü ne bu şu o için gibi daha en çok olarak olan olduğu ise ki ama veya hangi kim kimin neden
nasıl nerede ne zaman kaç her bazı tüm diğer sonra önce kadar göre üzerinde arasında tarafından`.split(/\s+/));
const WH_TIME = new Set(["when", "year", "zaman", "yıl", "yılında", "tarih", "century", "yüzyıl"]);
const WH_NUM = new Set(["many", "much", "kaç", "number", "percent", "yüzde", "sayı"]);
const WH_WHO = new Set(["who", "whom", "whose", "kim", "kimin", "kime", "kimdir"]);
const SENT_RE = /(?<=[.!?])\s+(?=[A-ZÇĞİÖŞÜ0-9"“(])/g;
const ALNUM = /[\p{L}\p{N}]/u;
const lowerTr = s => s.replace(/I/g, "ı").replace(/İ/g, "i").toLowerCase();
function splitSentences(text) {
text = text.replace(//g, " ");
const spans = []; let start = 0, m;
SENT_RE.lastIndex = 0;
while ((m = SENT_RE.exec(text)) !== null) {
if (text.slice(start, m.index).trim()) spans.push([start, m.index]);
start = m.index + m[0].length;
}
if (text.slice(start).trim()) spans.push([start, text.length]);
return spans;
}
function words(text) {
const out = []; let cur = "";
for (const ch of lowerTr(text)) { if (ALNUM.test(ch)) cur += ch; else if (cur) { out.push(cur); cur = ""; } }
if (cur) out.push(cur);
return out;
}
const stems = text => words(text).filter(w => !STOP.has(w)).map(w => Array.from(w).slice(0, 5).join(""));
function bm25(q, sents, k1 = 1.2, b = 0.75) {
const n = sents.length; let tot = 0; for (const s of sents) tot += s.length;
const avg = tot / Math.max(n, 1), df = new Map();
for (const s of sents) for (const t of new Set(s)) df.set(t, (df.get(t) || 0) + 1);
const idf = new Map(); for (const [t, d] of df) idf.set(t, Math.log(1.0 + (n - d + 0.5) / (d + 0.5)));
const scores = sents.map(s => {
const tf = new Map(); for (const t of s) tf.set(t, (tf.get(t) || 0) + 1);
let sc = 0.0;
for (const t of q) if (tf.has(t)) { const f = tf.get(t); sc += idf.get(t) * f * (k1 + 1) / (f + k1 * (1 - b + b * s.length / Math.max(avg, 1e-9))); }
return sc;
});
return [scores, idf];
}
function charNgrams(text, n = 4) {
const t = words(text).join(" "), out = new Set();
for (let i = 0; i < Math.max(0, t.length - n + 1); i++) out.add(t.slice(i, i + n));
return out;
}
const isUpper = c => c !== c.toLowerCase() && c === c.toUpperCase();
function sentenceFeatures(question, sents) {
const qSt = Array.from(new Set(stems(question))), qWords = new Set(words(question));
const sSt = sents.map(stems);
const [scores, idf] = bm25(qSt, sSt);
const mx = scores.length ? Math.max(...scores) : 0.0;
const order = scores.map((_, i) => i).sort((a, b) => (scores[b] - scores[a]) || (a - b));
const rank = new Map(order.map((i, r) => [i, r]));
const qIdf = qSt.map(t => (idf.has(t) ? idf.get(t) : Math.log(1.0 + (sents.length + 0.5) / 0.5)));
const qIdfSum = qIdf.reduce((a, b) => a + b, 0) || 1.0, qMax = qIdf.length ? Math.max(...qIdf) : 0;
const qNg = charNgrams(question);
const has = set => [...set].some(w => qWords.has(w));
const wantTime = has(WH_TIME), wantNum = has(WH_NUM), wantWho = has(WH_WHO);
const docSt = new Set(); for (const st of sSt) for (const t of st) docSt.add(t);
const docMissing = qSt.filter(t => !docSt.has(t)).length / Math.max(qSt.length, 1);
const qNums = words(question).filter(w => w && /^[0-9]+$/.test(w));
const bigQ = new Set(); for (let j = 0; j + 1 < qSt.length; j++) bigQ.add(qSt[j] + "\u0000" + qSt[j + 1]);
const feats = sents.map((s, i) => {
const ss = new Set(sSt[i]);
const matched = [], missing = [];
qSt.forEach((t, j) => (ss.has(t) ? matched : missing).push(qIdf[j]));
const ng = charNgrams(s); let inter = 0; for (const g of qNg) if (ng.has(g)) inter++;
const uni = qNg.size + ng.size - inter, jac = inter / Math.max(uni, 1);
const bigS = new Set(); for (let j = 0; j + 1 < sSt[i].length; j++) bigS.add(sSt[i][j] + "\u0000" + sSt[i][j + 1]);
let bi = 0; for (const g of bigQ) if (bigS.has(g)) bi++;
const hasNum = /[0-9]/.test(s);
const hasCap = s.split(/\s+/).filter(Boolean).slice(1).some(w => isUpper(w[0]));
const nb = [i - 1, i + 1].filter(j => j >= 0 && j < sents.length).map(j => scores[j]);
return [mx > 0 ? scores[i] / mx : 0.0, rank.get(i) / Math.max(sents.length - 1, 1), matched.length / Math.max(qSt.length, 1),
matched.reduce((a, b) => a + b, 0) / qIdfSum, matched.length && qIdf.length ? Math.max(...matched) / qMax : 0.0,
missing.length && qIdf.length ? Math.max(...missing) / qMax : 0.0, jac, Math.min(bi, 3) / 3.0,
(wantTime || wantNum) && hasNum ? 1.0 : 0.0, wantWho && hasCap ? 1.0 : 0.0, (wantTime || wantNum) && !hasNum ? 1.0 : 0.0,
Math.log(1 + sSt[i].length) / 4.0, i / Math.max(sents.length - 1, 1), nb.length && mx > 0 ? Math.max(...nb) / mx : 0.0,
Math.log(1 + mx) / 3.0, Math.log(1 + sents.length) / 4.0, docMissing,
qNums.length && qNums.some(n => !new Set(words(s)).has(n)) ? 1.0 : 0.0];
});
return [feats, scores];
}
const sigmoid = z => 1.0 / (1.0 + Math.exp(-Math.max(-50.0, Math.min(50.0, z))));
function scoreDocument(ctx, question, document) {
const spans = splitSentences(document), sents = spans.map(([a, b]) => document.slice(a, b));
if (!sents.length) return { spans: [], probs: [], bm25: [], answerable: 0.0, abstain: true, evidence: [] };
const [feats, scores] = sentenceFeatures(question, sents);
const probs = feats.map(f => sigmoid(NLRT.forward(ctx.m.nets.sent, NLRT.T([f.length], Float64Array.from(f))).data[0]));
const order = probs.map((_, i) => i).sort((a, b) => (probs[b] - probs[a]) || (a - b));
const top = order[0], p2 = order.length > 1 ? probs[order[1]] : 0.0;
const df = [probs[top], probs[top] - p2, feats[top][2], feats[top][3], feats[top][5], feats[top][10], feats[top][14], feats[top][16], feats[top][17]];
const w = ctx.m.tensors.ans_w.data; let z = 0; for (let i = 0; i < df.length; i++) z += w[i] * df[i];
const ans = sigmoid(z + ctx.m.tensors.ans_b.data[0]), th = ctx.m.meta.thresholds;
const abstain = ans < th.answerable;
const evidence = abstain ? [] : order.slice(0, 2).filter(i => i === top || probs[i] >= th.second_sentence);
return { spans, probs, bm25: scores, answerable: ans, abstain, evidence };
}
function bm25Baseline(ctx, question, document) {
const spans = splitSentences(document), sents = spans.map(([a, b]) => document.slice(a, b));
if (!sents.length) return { spans: [], abstain: true, evidence: [], bm25: [] };
const [scores] = bm25(Array.from(new Set(stems(question))), sents.map(stems));
let best = 0; for (let i = 1; i < scores.length; i++) if (scores[i] > scores[best]) best = i;
const abstain = scores[best] < ctx.m.meta.thresholds.bm25;
return { spans, abstain, evidence: abstain ? [] : [best], bm25: scores };
}
function init(assets) { return { m: NLRT.decode(assets["model.json"]) }; }
function runCase(ctx, input) {
const r = scoreDocument(ctx, input.question, input.document), b = bm25Baseline(ctx, input.question, input.document);
return { spans: r.spans, probs: r.probs, answerable: r.answerable, abstain: r.abstain, evidence: r.evidence, bm25_abstain: b.abstain, bm25_evidence: b.evidence };
}
return { ASSETS, init, splitSentences, scoreDocument, bm25Baseline, runCase, words, stems };
});
|