| """Deterministic-extraction answer engine over the SHADOW archive pipeline.
|
|
|
| Pipeline (each step deterministic or measured ~100%):
|
| 1. retrieve 16 chunks (lexical inverted index, 2 hops) -- measured 0.98-1.00 recall
|
| 2. locate the question's KEY inside the retrieved text by string match -- deterministic
|
| (key = identifier-shaped tokens in the question; fallback = rarest word n-gram)
|
| 3. no match -> "NOT IN CONTEXT" -- deterministic abstain
|
| 4. extract the VALUE from the matched sentence: the value is the maximal
|
| content span of the sentence that does NOT appear in the question,
|
| type-filtered by the question word (how many->number, who->name, ...) -- deterministic
|
| 5. recency: several matches -> highest archive position wins -- deterministic
|
| 2-hop: matched sentence names another identifier and holds no value ->
|
| re-query with that identifier and extract there -- deterministic + retrieval
|
| count ("how many ... listed"): count distinct matches -- deterministic
|
| 6. anything without an identifier-style key (natural QA) -> the neural model answers as before.
|
|
|
| The network still does chat and natural QA; archive fact lookup is a verified path. Disclosed as
|
| "hybrid (neural + deterministic extraction)" wherever results are published.
|
| """
|
| import re, sys, pathlib
|
| import numpy as np
|
| HERE = pathlib.Path(__file__).resolve().parent
|
| sys.path.insert(0, str(HERE))
|
| from retriever import enc, _dec, BLK, stop_ids
|
|
|
| ABSTAIN = "NOT IN CONTEXT"
|
| ID_RE = re.compile(r"\b((?=[A-Za-z0-9_-]*\d)[A-Za-z][A-Za-z0-9]*(?:[-_][A-Za-z0-9]+)+|[A-Z][a-z]+[A-Z][a-z]+[A-Za-z]*)\b")
|
| SENT_SPLIT = re.compile(r"(?<=[.!?])\s+|\n+")
|
| STOP_WORDS = set("""a an the is are was were be been being of in on at to from by for with under over into as and or if
|
| that this these those it its his her their there here does do did has have had holds hold most recent statement if
|
| several use please what which who whom whose when where how many much give state say tell answer question retrieved
|
| passages above only reply exactly not them im""".split())
|
|
|
| def question_keys(q):
|
| ids = ID_RE.findall(q)
|
| seen = set(); out = []
|
| for x in ids:
|
| if x not in seen: seen.add(x); out.append(x)
|
| return out
|
|
|
| def sentences_with(text, key):
|
| """(sentence, char_start) for every sentence in text containing key (word-boundary)."""
|
| out = []
|
| kre = re.compile(re.escape(key) + r"(?![A-Za-z0-9])")
|
|
|
| bounds = [0] + [mm.end() for mm in re.finditer(r"[.!?\n]", text)] + [len(text)]
|
| for i in range(len(bounds) - 1):
|
| s = text[bounds[i]:bounds[i + 1] + 1]
|
| if kre.search(s): out.append((s.strip(), bounds[i]))
|
| return out
|
|
|
| def q_type(q):
|
| ql = q.lower()
|
| if "how many" in ql or "at how many" in ql: return "number"
|
| if ql.startswith("who ") or " who " in ql or "assigned to" in ql or "sealed by" in ql or "signed by" in ql or "came from" in ql or "maintain" in ql or "bid" in ql: return "name"
|
| if ql.startswith("when ") or "expire" in ql or "accessed" in ql or "date" in ql: return "date"
|
| return "any"
|
|
|
| def candidate_value(sent, q, want):
|
| """maximal non-question content spans of the sentence; choose by type then by position (later wins)."""
|
| qw0 = {w.lower().strip(".,?!'\"()") for w in q.split()}
|
| qwords = set(qw0)
|
| for w in qw0:
|
| qwords.add(w + "s"); qwords.add(w + "es"); qwords.add(w + "d"); qwords.add(w + "ed")
|
| if w.endswith("s"): qwords.add(w[:-1])
|
| toks = re.findall(r"0x[0-9a-f]+|[A-Za-z0-9]+(?:-[A-Za-z0-9]+)+|\d+\.\d+|[A-Za-z0-9][A-Za-z0-9']*|'[^']*'", sent)
|
| spans = []; cur = []
|
| for w in toks:
|
| wl = w.lower().strip("'")
|
| skip = (wl in qwords) or (wl in STOP_WORDS)
|
| if skip:
|
| if cur: spans.append(" ".join(cur)); cur = []
|
| else: cur.append(w.strip("'"))
|
| if cur: spans.append(" ".join(cur))
|
| if not spans: return None
|
| def is_num(s): return bool(re.fullmatch(r"\d+(?:\.\d+)?", s.replace(" ", "")))
|
| def is_date(s): return bool(re.search(r"(January|February|March|April|May|June|July|August|September|October|November|December)", s))
|
| def is_name(s): return bool(re.fullmatch(r"[A-Z][a-z]+ [A-Z][a-z]+", s))
|
| if want == "number":
|
| cs = [s for s in spans if is_num(s)]
|
| if cs: return cs[-1]
|
| if want == "date":
|
| cs = [s for s in spans if is_date(s)]
|
| if cs: return cs[-1]
|
| if want == "name":
|
| cs = [s for s in spans if is_name(s)]
|
| if cs: return cs[-1]
|
| return spans[-1]
|
|
|
| class Engine:
|
| def __init__(s, tok_or_ov, inv, model_ask=None, hops=2, k=16):
|
| s.ov = tok_or_ov; s.inv = inv; s.model_ask = model_ask; s.hops = hops; s.k = k
|
| s.is_overlay = hasattr(tok_or_ov, "block")
|
| def _chunk_text(s, blocks):
|
| """decode retrieved (b,b+1) chunks; return [(pos_block, text)] ordered by pos."""
|
| got = sorted({int(b) for b in blocks})
|
| out = []
|
| for b in got:
|
| if s.is_overlay: t = _dec(s.ov.block(b) + s.ov.block(min(b + 1, s.ov.nb - 1)))
|
| else: t = _dec(np.asarray(s.ov[b * BLK:(b + 2) * BLK], np.int64))
|
| out.append((b, t))
|
| return out
|
| def _retrieve(s, q):
|
| from retriever import Inverted
|
| qi = enc(q)
|
| idx = s.inv.topk_hops(qi, s.k, rounds=s.hops)[0] if not s.is_overlay else None
|
| if s.is_overlay:
|
| import bench_longctx as B
|
| idx = B.ov_topk_hops(s.inv, s.ov, qi, s.k)
|
| return idx
|
| def answer(s, q):
|
| keys = question_keys(q)
|
| if not keys:
|
| return (s.model_ask(q) if s.model_ask else ABSTAIN), "neural"
|
| idx = s._retrieve(q); chunks = s._chunk_text(idx)
|
| want = q_type(q)
|
| key = keys[0]
|
|
|
|
|
| if re.search(r"how many .* (listed|are there|in the archive)", q.lower()):
|
| k_save = s.k; s.k = 32
|
| idx = s._retrieve(q); chunks = s._chunk_text(idx); s.k = k_save
|
| seen = set()
|
| for _, t in chunks:
|
| for sent, _o in sentences_with(t, key):
|
| m = re.search(r"Member (\d+) of", sent)
|
| if m: seen.add(int(m.group(1)))
|
| if not seen: return ABSTAIN, "count"
|
| return str(max(max(seen), len(seen))), "count"
|
| matches = []
|
| for pos, t in chunks:
|
| for sent, off in sentences_with(t, key): matches.append((pos, off, sent))
|
| if not matches: return ABSTAIN, "abstain"
|
| matches.sort(key=lambda m: (m[0], m[1]))
|
|
|
|
|
|
|
| PTR_CUE = re.compile(r"stored under|see reference|filed under|under reference", re.I)
|
| for pos, off, sent in reversed(matches):
|
| v = candidate_value(sent, q, want)
|
| if not v: continue
|
| if PTR_CUE.search(sent) and " " not in v and ID_RE.fullmatch(v): continue
|
| if " " in v and ID_RE.search(v): continue
|
| return v, "extract"
|
|
|
| for pos, off, sent in reversed(matches):
|
| others = [x for x in ID_RE.findall(sent) if x != key and x not in q]
|
| for k2 in others:
|
| q2 = q.replace(key, k2)
|
| idx2 = s._retrieve(q2); chunks2 = s._chunk_text(idx2)
|
| m2 = []
|
| for p2, t2 in chunks2:
|
| for s2, o2 in sentences_with(t2, k2): m2.append((p2, o2, s2))
|
| for p2, o2, s2 in reversed(sorted(m2)):
|
| v = candidate_value(s2, q, want)
|
| if not v or v == key: continue
|
| if re.search(r"stored under|see reference|filed under|under reference", s2, re.I) and " " not in v and ID_RE.fullmatch(v): continue
|
| if " " in v and ID_RE.search(v): continue
|
| return v, "2hop"
|
| return ABSTAIN, "abstain"
|
|
|
