File size: 8,205 Bytes
c07bca1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
/* Next Trace compute core: line-by-line port of trace_core.py. */
(function (root, factory) {
  if (typeof module === "object" && module.exports) module.exports = factory();
  else root.Core = factory();
})(typeof self !== "undefined" ? self : this, function () {
  "use strict";
  const ASSETS = ["model.json"];
  const STOP = new Set(`a an the of in on at to for from by with and or but is are was were be been being it its this that these those as
which who whom whose what when where why how do does did has have had not no than then there their they he she his her
we you your i me my our us can could would should will may might also into over under about after before between during
bir ve ile de da mi mı mu mü ne bu şu o için gibi daha en çok olarak olan olduğu ise ki ama veya hangi kim kimin neden
nasıl nerede ne zaman kaç her bazı tüm diğer sonra önce kadar göre üzerinde arasında tarafından`.split(/\s+/));
  const WH_TIME = new Set(["when", "year", "zaman", "yıl", "yılında", "tarih", "century", "yüzyıl"]);
  const WH_NUM = new Set(["many", "much", "kaç", "number", "percent", "yüzde", "sayı"]);
  const WH_WHO = new Set(["who", "whom", "whose", "kim", "kimin", "kime", "kimdir"]);
  const SENT_RE = /(?<=[.!?])\s+(?=[A-ZÇĞİÖŞÜ0-9"“(])/g;
  const ALNUM = /[\p{L}\p{N}]/u;

  const lowerTr = s => s.replace(/I/g, "ı").replace(/İ/g, "i").toLowerCase();

  function splitSentences(text) {
    text = text.replace(//g, " ");
    const spans = []; let start = 0, m;
    SENT_RE.lastIndex = 0;
    while ((m = SENT_RE.exec(text)) !== null) {
      if (text.slice(start, m.index).trim()) spans.push([start, m.index]);
      start = m.index + m[0].length;
    }
    if (text.slice(start).trim()) spans.push([start, text.length]);
    return spans;
  }

  function words(text) {
    const out = []; let cur = "";
    for (const ch of lowerTr(text)) { if (ALNUM.test(ch)) cur += ch; else if (cur) { out.push(cur); cur = ""; } }
    if (cur) out.push(cur);
    return out;
  }
  const stems = text => words(text).filter(w => !STOP.has(w)).map(w => Array.from(w).slice(0, 5).join(""));

  function bm25(q, sents, k1 = 1.2, b = 0.75) {
    const n = sents.length; let tot = 0; for (const s of sents) tot += s.length;
    const avg = tot / Math.max(n, 1), df = new Map();
    for (const s of sents) for (const t of new Set(s)) df.set(t, (df.get(t) || 0) + 1);
    const idf = new Map(); for (const [t, d] of df) idf.set(t, Math.log(1.0 + (n - d + 0.5) / (d + 0.5)));
    const scores = sents.map(s => {
      const tf = new Map(); for (const t of s) tf.set(t, (tf.get(t) || 0) + 1);
      let sc = 0.0;
      for (const t of q) if (tf.has(t)) { const f = tf.get(t); sc += idf.get(t) * f * (k1 + 1) / (f + k1 * (1 - b + b * s.length / Math.max(avg, 1e-9))); }
      return sc;
    });
    return [scores, idf];
  }

  function charNgrams(text, n = 4) {
    const t = words(text).join(" "), out = new Set();
    for (let i = 0; i < Math.max(0, t.length - n + 1); i++) out.add(t.slice(i, i + n));
    return out;
  }
  const isUpper = c => c !== c.toLowerCase() && c === c.toUpperCase();

  function sentenceFeatures(question, sents) {
    const qSt = Array.from(new Set(stems(question))), qWords = new Set(words(question));
    const sSt = sents.map(stems);
    const [scores, idf] = bm25(qSt, sSt);
    const mx = scores.length ? Math.max(...scores) : 0.0;
    const order = scores.map((_, i) => i).sort((a, b) => (scores[b] - scores[a]) || (a - b));
    const rank = new Map(order.map((i, r) => [i, r]));
    const qIdf = qSt.map(t => (idf.has(t) ? idf.get(t) : Math.log(1.0 + (sents.length + 0.5) / 0.5)));
    const qIdfSum = qIdf.reduce((a, b) => a + b, 0) || 1.0, qMax = qIdf.length ? Math.max(...qIdf) : 0;
    const qNg = charNgrams(question);
    const has = set => [...set].some(w => qWords.has(w));
    const wantTime = has(WH_TIME), wantNum = has(WH_NUM), wantWho = has(WH_WHO);
    const docSt = new Set(); for (const st of sSt) for (const t of st) docSt.add(t);
    const docMissing = qSt.filter(t => !docSt.has(t)).length / Math.max(qSt.length, 1);
    const qNums = words(question).filter(w => w && /^[0-9]+$/.test(w));
    const bigQ = new Set(); for (let j = 0; j + 1 < qSt.length; j++) bigQ.add(qSt[j] + "\u0000" + qSt[j + 1]);
    const feats = sents.map((s, i) => {
      const ss = new Set(sSt[i]);
      const matched = [], missing = [];
      qSt.forEach((t, j) => (ss.has(t) ? matched : missing).push(qIdf[j]));
      const ng = charNgrams(s); let inter = 0; for (const g of qNg) if (ng.has(g)) inter++;
      const uni = qNg.size + ng.size - inter, jac = inter / Math.max(uni, 1);
      const bigS = new Set(); for (let j = 0; j + 1 < sSt[i].length; j++) bigS.add(sSt[i][j] + "\u0000" + sSt[i][j + 1]);
      let bi = 0; for (const g of bigQ) if (bigS.has(g)) bi++;
      const hasNum = /[0-9]/.test(s);
      const hasCap = s.split(/\s+/).filter(Boolean).slice(1).some(w => isUpper(w[0]));
      const nb = [i - 1, i + 1].filter(j => j >= 0 && j < sents.length).map(j => scores[j]);
      return [mx > 0 ? scores[i] / mx : 0.0, rank.get(i) / Math.max(sents.length - 1, 1), matched.length / Math.max(qSt.length, 1),
        matched.reduce((a, b) => a + b, 0) / qIdfSum, matched.length && qIdf.length ? Math.max(...matched) / qMax : 0.0,
        missing.length && qIdf.length ? Math.max(...missing) / qMax : 0.0, jac, Math.min(bi, 3) / 3.0,
        (wantTime || wantNum) && hasNum ? 1.0 : 0.0, wantWho && hasCap ? 1.0 : 0.0, (wantTime || wantNum) && !hasNum ? 1.0 : 0.0,
        Math.log(1 + sSt[i].length) / 4.0, i / Math.max(sents.length - 1, 1), nb.length && mx > 0 ? Math.max(...nb) / mx : 0.0,
        Math.log(1 + mx) / 3.0, Math.log(1 + sents.length) / 4.0, docMissing,
        qNums.length && qNums.some(n => !new Set(words(s)).has(n)) ? 1.0 : 0.0];
    });
    return [feats, scores];
  }

  const sigmoid = z => 1.0 / (1.0 + Math.exp(-Math.max(-50.0, Math.min(50.0, z))));

  function scoreDocument(ctx, question, document) {
    const spans = splitSentences(document), sents = spans.map(([a, b]) => document.slice(a, b));
    if (!sents.length) return { spans: [], probs: [], bm25: [], answerable: 0.0, abstain: true, evidence: [] };
    const [feats, scores] = sentenceFeatures(question, sents);
    const probs = feats.map(f => sigmoid(NLRT.forward(ctx.m.nets.sent, NLRT.T([f.length], Float64Array.from(f))).data[0]));
    const order = probs.map((_, i) => i).sort((a, b) => (probs[b] - probs[a]) || (a - b));
    const top = order[0], p2 = order.length > 1 ? probs[order[1]] : 0.0;
    const df = [probs[top], probs[top] - p2, feats[top][2], feats[top][3], feats[top][5], feats[top][10], feats[top][14], feats[top][16], feats[top][17]];
    const w = ctx.m.tensors.ans_w.data; let z = 0; for (let i = 0; i < df.length; i++) z += w[i] * df[i];
    const ans = sigmoid(z + ctx.m.tensors.ans_b.data[0]), th = ctx.m.meta.thresholds;
    const abstain = ans < th.answerable;
    const evidence = abstain ? [] : order.slice(0, 2).filter(i => i === top || probs[i] >= th.second_sentence);
    return { spans, probs, bm25: scores, answerable: ans, abstain, evidence };
  }

  function bm25Baseline(ctx, question, document) {
    const spans = splitSentences(document), sents = spans.map(([a, b]) => document.slice(a, b));
    if (!sents.length) return { spans: [], abstain: true, evidence: [], bm25: [] };
    const [scores] = bm25(Array.from(new Set(stems(question))), sents.map(stems));
    let best = 0; for (let i = 1; i < scores.length; i++) if (scores[i] > scores[best]) best = i;
    const abstain = scores[best] < ctx.m.meta.thresholds.bm25;
    return { spans, abstain, evidence: abstain ? [] : [best], bm25: scores };
  }

  function init(assets) { return { m: NLRT.decode(assets["model.json"]) }; }

  function runCase(ctx, input) {
    const r = scoreDocument(ctx, input.question, input.document), b = bm25Baseline(ctx, input.question, input.document);
    return { spans: r.spans, probs: r.probs, answerable: r.answerable, abstain: r.abstain, evidence: r.evidence, bm25_abstain: b.abstain, bm25_evidence: b.evidence };
  }

  return { ASSETS, init, splitSentences, scoreDocument, bm25Baseline, runCase, words, stems };
});