Download trace_core.py from thelamapi/Next-Trace: direct link, hf CLI and curl.
- Browser
- Download file 7.64 kB
-
https://huggingface.co/thelamapi/Next-Trace/resolve/main/trace_core.py
- Command line
-
hf download hf://thelamapi/Next-Trace/trace_core.py
-
curl -L -o trace_core.py https://huggingface.co/thelamapi/Next-Trace/resolve/main/trace_core.py
7.64 kB
| """Next Trace: evidence sentences for a question over a Turkish or English document, or an explicit abstention. | |
| Pure Python + nlrt (NumPy). Ported line by line to web/core.js. | |
| * split_sentences / tokens: language-light text processing (Turkish-aware lower-casing, 5-letter stems) | |
| * bm25: Okapi BM25 over the document's sentences | |
| * sentence_features: 18 features per (question, sentence) pair | |
| * score_document: evidence probability per sentence (MLP) + answerability probability (logistic on document features) | |
| """ | |
| import math | |
| import re | |
| import numpy as np | |
| from nlrt import forward | |
| STOP = set("""a an the of in on at to for from by with and or but is are was were be been being it its this that these those as | |
| which who whom whose what when where why how do does did has have had not no than then there their they he she his her | |
| we you your i me my our us can could would should will may might also into over under about after before between during | |
| bir ve ile de da mi mı mu mü ne bu şu o için gibi daha en çok olarak olan olduğu ise ki ama veya hangi kim kimin neden | |
| nasıl nerede ne zaman kaç her bazı tüm diğer sonra önce kadar göre üzerinde arasında tarafından""".split()) | |
| WH_TIME = {"when", "year", "zaman", "yıl", "yılında", "tarih", "century", "yüzyıl"} | |
| WH_NUM = {"many", "much", "kaç", "number", "percent", "yüzde", "sayı"} | |
| WH_WHO = {"who", "whom", "whose", "kim", "kimin", "kime", "kimdir"} | |
| SENT_RE = re.compile(r"(?<=[.!?])\s+(?=[A-ZÇĞİÖŞÜ0-9\"“(])") | |
| N_FEAT = 18 | |
| def lower_tr(s): | |
| return s.replace("I", "ı").replace("İ", "i").lower() | |
| def split_sentences(text): | |
| """Returns list of (start, end) character spans.""" | |
| text = text.replace("", " ") | |
| spans, start = [], 0 | |
| for m in SENT_RE.finditer(text): | |
| if text[start:m.start()].strip(): | |
| spans.append((start, m.start())) | |
| start = m.end() | |
| if text[start:].strip(): | |
| spans.append((start, len(text))) | |
| return spans | |
| def words(text): | |
| out, cur = [], [] | |
| for ch in lower_tr(text): | |
| if ch.isalnum(): | |
| cur.append(ch) | |
| elif cur: | |
| out.append("".join(cur)); cur = [] | |
| if cur: | |
| out.append("".join(cur)) | |
| return out | |
| def stems(text): | |
| return [w[:5] for w in words(text) if w not in STOP] | |
| def bm25(q_stems, sent_stems, k1=1.2, b=0.75): | |
| n = len(sent_stems) | |
| avg = sum(len(s) for s in sent_stems) / max(n, 1) | |
| df = {} | |
| for s in sent_stems: | |
| for t in set(s): | |
| df[t] = df.get(t, 0) + 1 | |
| idf = {t: math.log(1.0 + (n - d + 0.5) / (d + 0.5)) for t, d in df.items()} | |
| scores = [] | |
| for s in sent_stems: | |
| tf = {} | |
| for t in s: | |
| tf[t] = tf.get(t, 0) + 1 | |
| sc = 0.0 | |
| for t in q_stems: | |
| if t in tf: | |
| f = tf[t] | |
| sc += idf[t] * f * (k1 + 1) / (f + k1 * (1 - b + b * len(s) / max(avg, 1e-9))) | |
| scores.append(sc) | |
| return scores, idf | |
| def char_ngrams(text, n=4): | |
| t = " ".join(words(text)) | |
| return set(t[i:i + n] for i in range(max(0, len(t) - n + 1))) | |
| def sentence_features(question, sents): | |
| """sents: list of sentence strings. Returns (features per sentence, bm25 scores).""" | |
| q_st = list(dict.fromkeys(stems(question))) | |
| q_words = set(words(question)) | |
| s_st = [stems(s) for s in sents] | |
| scores, idf = bm25(q_st, s_st) | |
| mx = max(scores) if scores else 0.0 | |
| order = sorted(range(len(scores)), key=lambda i: (-scores[i], i)) | |
| rank = {i: r for r, i in enumerate(order)} | |
| q_idf = [idf.get(t, math.log(1.0 + (len(sents) + 0.5) / 0.5)) for t in q_st] | |
| q_idf_sum = sum(q_idf) or 1.0 | |
| q_ng = char_ngrams(question) | |
| want_time, want_num, want_who = bool(q_words & WH_TIME), bool(q_words & WH_NUM), bool(q_words & WH_WHO) | |
| doc_st = set(t for st in s_st for t in st) | |
| doc_missing = sum(1 for t in q_st if t not in doc_st) / max(len(q_st), 1) | |
| q_nums = [w for w in words(question) if w and all(ch in "0123456789" for ch in w)] | |
| feats = [] | |
| for i, s in enumerate(sents): | |
| ss = set(s_st[i]) | |
| matched = [q_idf[j] for j, t in enumerate(q_st) if t in ss] | |
| missing = [q_idf[j] for j, t in enumerate(q_st) if t not in ss] | |
| ng = char_ngrams(s) | |
| jac = len(q_ng & ng) / max(len(q_ng | ng), 1) | |
| bigr_q = set(zip(q_st, q_st[1:])) | |
| bigr_s = set(zip(s_st[i], s_st[i][1:])) | |
| has_num = any(ch in "0123456789" for ch in s) | |
| has_cap = sum(1 for w in s.split()[1:] if w[:1].isupper()) > 0 | |
| nb = [scores[j] for j in (i - 1, i + 1) if 0 <= j < len(sents)] | |
| feats.append([ | |
| scores[i] / mx if mx > 0 else 0.0, | |
| rank[i] / max(len(sents) - 1, 1), | |
| len(matched) / max(len(q_st), 1), | |
| sum(matched) / q_idf_sum, | |
| max(matched) / max(q_idf) if matched and q_idf else 0.0, | |
| max(missing) / max(q_idf) if missing and q_idf else 0.0, | |
| jac, | |
| min(len(bigr_q & bigr_s), 3) / 3.0, | |
| 1.0 if (want_time or want_num) and has_num else 0.0, | |
| 1.0 if want_who and has_cap else 0.0, | |
| 1.0 if (want_time or want_num) and not has_num else 0.0, | |
| math.log(1 + len(s_st[i])) / 4.0, | |
| i / max(len(sents) - 1, 1), | |
| (max(nb) / mx) if nb and mx > 0 else 0.0, | |
| math.log(1 + mx) / 3.0, | |
| math.log(1 + len(sents)) / 4.0, | |
| doc_missing, | |
| 1.0 if q_nums and any(n not in set(words(s)) for n in q_nums) else 0.0, | |
| ]) | |
| return feats, scores | |
| def doc_features(probs, feats): | |
| """Answerability features from sentence probabilities and the top sentence's features.""" | |
| order = sorted(range(len(probs)), key=lambda i: (-probs[i], i)) | |
| top = order[0] | |
| p2 = probs[order[1]] if len(order) > 1 else 0.0 | |
| return [probs[top], probs[top] - p2, feats[top][2], feats[top][3], feats[top][5], feats[top][10], feats[top][14], feats[top][16], feats[top][17]] | |
| def sigmoid(z): | |
| return 1.0 / (1.0 + math.exp(-max(-50.0, min(50.0, z)))) | |
| def score_document(model, question, document): | |
| """model: dict with nets['sent'] (MLP) and tensors ans_w, ans_b, meta thresholds. Returns a result dict.""" | |
| spans = split_sentences(document) | |
| sents = [document[a:b] for a, b in spans] | |
| if not sents: | |
| return {"spans": [], "probs": [], "bm25": [], "answerable": 0.0, "abstain": True, "evidence": []} | |
| feats, scores = sentence_features(question, sents) | |
| z = forward(model["nets"]["sent"], np.asarray(feats))[:, 0] | |
| probs = [sigmoid(v) for v in z] | |
| df = doc_features(probs, feats) | |
| w = model["tensors"]["ans_w"] | |
| b = float(model["tensors"]["ans_b"][0]) | |
| ans = sigmoid(sum(float(w[i]) * df[i] for i in range(len(df))) + b) | |
| th = model["meta"]["thresholds"] | |
| order = sorted(range(len(probs)), key=lambda i: (-probs[i], i)) | |
| abstain = ans < th["answerable"] | |
| evidence = [] if abstain else [i for i in order[:2] if i == order[0] or probs[i] >= th["second_sentence"]] | |
| return {"spans": spans, "probs": probs, "bm25": scores, "answerable": ans, "abstain": abstain, "evidence": evidence} | |
| def bm25_baseline(question, document, threshold): | |
| spans = split_sentences(document) | |
| sents = [document[a:b] for a, b in spans] | |
| if not sents: | |
| return {"spans": [], "abstain": True, "evidence": [], "bm25": []} | |
| q_st = list(dict.fromkeys(stems(question))) | |
| scores, _ = bm25(q_st, [stems(s) for s in sents]) | |
| best = max(range(len(scores)), key=lambda i: (scores[i], -i)) | |
| abstain = scores[best] < threshold | |
| return {"spans": spans, "abstain": abstain, "evidence": [] if abstain else [best], "bm25": scores} | |